From e5ebd28ee12fc06465441936e895db03e667b98f Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 12:34:25 -0400 Subject: [PATCH 01/47] Added get_new_service_instance_stub that creates a new service instance stub --- salt/utils/vmware.py | 52 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/salt/utils/vmware.py b/salt/utils/vmware.py index d54dbced0421..cbfb741dc0fc 100644 --- a/salt/utils/vmware.py +++ b/salt/utils/vmware.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- ''' +import sys +import ssl Connection library for VMware .. versionadded:: 2015.8.2 @@ -79,6 +81,8 @@ import errno import logging import time +import sys +import ssl # Import Salt Libs import salt.exceptions @@ -92,8 +96,9 @@ from salt.ext import six from salt.ext.six.moves.http_client import BadStatusLine # pylint: disable=E0611 try: - from pyVim.connect import GetSi, SmartConnect, Disconnect, GetStub - from pyVmomi import vim, vmodl + from pyVim.connect import GetSi, SmartConnect, Disconnect, GetStub, \ + SoapStubAdapter + from pyVmomi import vim, vmodl, VmomiSupport HAS_PYVMOMI = True except ImportError: HAS_PYVMOMI = False @@ -405,6 +410,49 @@ def get_service_instance(host, username=None, password=None, protocol=None, return service_instance +def get_new_service_instance_stub(service_instance, path, ns=None, + version=None): + ''' + Returns a stub that points to a different path, + created from an existing connection. + + service_instance + The Service Instance. + + path + Path of the new stub. + + ns + Namespace of the new stub. + Default value is None + + version + Version of the new stub. + Default value is None. + ''' + #For python 2.7.9 and later, the defaul SSL conext has more strict + #connection handshaking rule. We may need turn of the hostname checking + #and client side cert verification + context = None + if sys.version_info[:3] > (2,7,8): + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + stub = service_instance._stub + hostname = stub.host.split(':')[0] + session_cookie = stub.cookie.split('"')[1] + VmomiSupport.GetRequestContext()['vcSessionCookie'] = session_cookie + new_stub = SoapStubAdapter(host=hostname, + ns=ns, + path=path, + version=version, + poolSize=0, + sslContext=context) + new_stub.cookie = stub.cookie + return new_stub + + def get_service_instance_from_managed_object(mo_ref, name=''): ''' Retrieves the service instance from a managed object. From dd54f8ab15fc5038d356b49cb8b945e04344bff2 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 13:16:58 -0400 Subject: [PATCH 02/47] Added tests for salt.utils.vmware.get_new_service_instance_stub --- tests/unit/utils/vmware/test_connection.py | 93 +++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/vmware/test_connection.py b/tests/unit/utils/vmware/test_connection.py index 4a95e9b67fc2..dd357d487087 100644 --- a/tests/unit/utils/vmware/test_connection.py +++ b/tests/unit/utils/vmware/test_connection.py @@ -13,6 +13,7 @@ import sys # Import Salt testing libraries +from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock, call, \ PropertyMock @@ -24,7 +25,7 @@ from salt.ext import six try: - from pyVmomi import vim, vmodl + from pyVmomi import vim, vmodl, VmomiSupport HAS_PYVMOMI = True except ImportError: HAS_PYVMOMI = False @@ -852,6 +853,96 @@ def test_connected_to_invalid_entity(self): excinfo.exception.strerror) +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetNewServiceInstanceStub(TestCase, LoaderModuleMockMixin): + '''Tests for salt.utils.vmware.get_new_service_instance_stub''' + def setup_loader_modules(self): + return {salt.utils.vmware: { + '__virtual__': MagicMock(return_value='vmware'), + 'sys': MagicMock(), + 'ssl': MagicMock()}} + + def setUp(self): + self.mock_stub = MagicMock( + host='fake_host:1000', + cookie='ignore"fake_cookie') + self.mock_si = MagicMock( + _stub=self.mock_stub) + self.mock_ret = MagicMock() + self.mock_new_stub = MagicMock() + self.context_dict = {} + patches = (('salt.utils.vmware.VmomiSupport.GetRequestContext', + MagicMock( + return_value=self.context_dict)), + ('salt.utils.vmware.SoapStubAdapter', + MagicMock(return_value=self.mock_new_stub))) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + type(salt.utils.vmware.sys).version_info = \ + PropertyMock(return_value=(2, 7, 9)) + self.mock_context = MagicMock() + self.mock_create_default_context = \ + MagicMock(return_value=self.mock_context) + salt.utils.vmware.ssl.create_default_context = \ + self.mock_create_default_context + + def tearDown(self): + for attr in ('mock_stub', 'mock_si', 'mock_ret', 'mock_new_stub', + 'context_dict', 'mock_context', + 'mock_create_default_context'): + delattr(self, attr) + + def test_ssl_default_context_loaded(self): + salt.utils.vmware.get_new_service_instance_stub( + self.mock_si, 'fake_path') + self.mock_create_default_context.assert_called_once_with() + self.assertFalse(self.mock_context.check_hostname) + self.assertEqual(self.mock_context.verify_mode, + salt.utils.vmware.ssl.CERT_NONE) + + def test_ssl_default_context_not_loaded(self): + type(salt.utils.vmware.sys).version_info = \ + PropertyMock(return_value=(2, 7, 8)) + salt.utils.vmware.get_new_service_instance_stub( + self.mock_si, 'fake_path') + self.assertEqual(self.mock_create_default_context.call_count, 0) + + def test_session_cookie_in_context(self): + salt.utils.vmware.get_new_service_instance_stub( + self.mock_si, 'fake_path') + self.assertEqual(self.context_dict['vcSessionCookie'], 'fake_cookie') + + def test_get_new_stub(self): + mock_get_new_stub = MagicMock() + with patch('salt.utils.vmware.SoapStubAdapter', mock_get_new_stub): + salt.utils.vmware.get_new_service_instance_stub( + self.mock_si, 'fake_path', 'fake_ns', 'fake_version') + mock_get_new_stub.assert_called_once_with( + host='fake_host', ns='fake_ns', path='fake_path', + version='fake_version', poolSize=0, sslContext=self.mock_context) + + def test_get_new_stub_2_7_8_python(self): + type(salt.utils.vmware.sys).version_info = \ + PropertyMock(return_value=(2, 7, 8)) + mock_get_new_stub = MagicMock() + with patch('salt.utils.vmware.SoapStubAdapter', mock_get_new_stub): + salt.utils.vmware.get_new_service_instance_stub( + self.mock_si, 'fake_path', 'fake_ns', 'fake_version') + mock_get_new_stub.assert_called_once_with( + host='fake_host', ns='fake_ns', path='fake_path', + version='fake_version', poolSize=0, sslContext=None) + + def test_new_stub_returned(self): + ret = salt.utils.vmware.get_new_service_instance_stub( + self.mock_si, 'fake_path') + self.assertEqual(self.mock_new_stub.cookie, 'ignore"fake_cookie') + self.assertEqual(ret, self.mock_new_stub) + + @skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') class GetServiceInstanceFromManagedObjectTestCase(TestCase): From 3e8ed5934d97e33a2dd5f1d19841b4e15cb87b16 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 13:33:12 -0400 Subject: [PATCH 03/47] Added initial sysdoc and imports of salt.utils.pbm --- salt/utils/pbm.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 salt/utils/pbm.py diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py new file mode 100644 index 000000000000..9d9e7bb9898b --- /dev/null +++ b/salt/utils/pbm.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +''' +Library for VMware Storage Policy management (via the pbm endpoint) + +This library is used to manage the various policies available in VMware + +:codeauthor: Alexandru Bleotu + +Dependencies +~~~~~~~~~~~~ + +- pyVmomi Python Module + +pyVmomi +------- + +PyVmomi can be installed via pip: + +.. code-block:: bash + + pip install pyVmomi + +.. note:: + + versions of Python. If using version 6.0 of pyVmomi, Python 2.6, + Python 2.7.9, or newer must be present. This is due to an upstream dependency + in pyVmomi 6.0 that is not supported in Python versions 2.7 to 2.7.8. If the + version of Python is not in the supported range, you will need to install an + earlier version of pyVmomi. See `Issue #29537`_ for more information. + +.. _Issue #29537: https://github.com/saltstack/salt/issues/29537 + +Based on the note above, to install an earlier version of pyVmomi than the +version currently listed in PyPi, run the following: + +.. code-block:: bash + + pip install pyVmomi==5.5.0.2014.1.1 +''' + +# Import Python Libs +from __future__ import absolute_import +import logging + +# Import Salt Libs +import salt.utils.vmware +from salt.exceptions import VMwareApiError, VMwareRuntimeError, \ + VMwareObjectRetrievalError + + +try: + from pyVmomi import pbm, vim, vmodl + HAS_PYVMOMI = True +except ImportError: + HAS_PYVMOMI = False + + +# Get Logging Started +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only load if PyVmomi is installed. + ''' + if HAS_PYVMOMI: + return True + else: + return False, 'Missing dependency: The salt.utils.pbm module ' \ + 'requires the pyvmomi library' From e77b912f2cb65b1901f2a297386b50f55a826dc8 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 13:34:33 -0400 Subject: [PATCH 04/47] Added salt.utils.pbm.get_profile_manager --- salt/utils/pbm.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index 9d9e7bb9898b..aec534111224 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -68,3 +68,28 @@ def __virtual__(): else: return False, 'Missing dependency: The salt.utils.pbm module ' \ 'requires the pyvmomi library' + + +def get_profile_manager(service_instance): + ''' + Returns a profile manager + + service_instance + Service instance to the host or vCenter + ''' + stub = salt.utils.vmware.get_new_service_instance_stub( + service_instance, ns='pbm/2.0', path='/pbm/sdk') + pbm_si = pbm.ServiceInstance('ServiceInstance', stub) + try: + profile_manager = pbm_si.RetrieveContent().profileManager + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) + return profile_manager From 6b2ddffb4c7a0585ddbce6d4c9fad8c7c150fc97 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 13:35:04 -0400 Subject: [PATCH 05/47] Added tests for salt.utils.pbm.get_profile_manager --- tests/unit/utils/test_pbm.py | 105 +++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tests/unit/utils/test_pbm.py diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py new file mode 100644 index 000000000000..11256c9b32e2 --- /dev/null +++ b/tests/unit/utils/test_pbm.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu ` + + Tests functions in salt.utils.vsan +''' + +# Import python libraries +from __future__ import absolute_import +import logging + +# Import Salt testing libraries +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock, \ + PropertyMock + +# Import Salt libraries +from salt.exceptions import VMwareApiError, VMwareRuntimeError +import salt.utils.pbm + +try: + from pyVmomi import vim, vmodl, pbm + HAS_PYVMOMI = True +except ImportError: + HAS_PYVMOMI = False + + +# Get Logging Started +log = logging.getLogger(__name__) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetProfileManagerTestCase(TestCase): + '''Tests for salt.utils.pbm.get_profile_manager''' + def setUp(self): + self.mock_si = MagicMock() + self.mock_stub = MagicMock() + self.mock_prof_mgr = MagicMock() + self.mock_content = MagicMock() + self.mock_pbm_si = MagicMock( + RetrieveContent=MagicMock(return_value=self.mock_content)) + type(self.mock_content).profileManager = \ + PropertyMock(return_value=self.mock_prof_mgr) + patches = ( + ('salt.utils.vmware.get_new_service_instance_stub', + MagicMock(return_value=self.mock_stub)), + ('salt.utils.pbm.pbm.ServiceInstance', + MagicMock(return_value=self.mock_pbm_si))) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attr in ('mock_si', 'mock_stub', 'mock_content', + 'mock_pbm_si', 'mock_prof_mgr'): + delattr(self, attr) + + def test_get_new_service_stub(self): + mock_get_new_service_stub = MagicMock() + with patch('salt.utils.vmware.get_new_service_instance_stub', + mock_get_new_service_stub): + salt.utils.pbm.get_profile_manager(self.mock_si) + mock_get_new_service_stub.assert_called_once_with( + self.mock_si, ns='pbm/2.0', path='/pbm/sdk') + + def test_pbm_si(self): + mock_get_pbm_si = MagicMock() + with patch('salt.utils.pbm.pbm.ServiceInstance', + mock_get_pbm_si): + salt.utils.pbm.get_profile_manager(self.mock_si) + mock_get_pbm_si.assert_called_once_with('ServiceInstance', + self.mock_stub) + + def test_return_profile_manager(self): + ret = salt.utils.pbm.get_profile_manager(self.mock_si) + self.assertEqual(ret, self.mock_prof_mgr) + + def test_profile_manager_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + type(self.mock_content).profileManager = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_profile_manager(self.mock_si) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_profile_manager_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + type(self.mock_content).profileManager = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_profile_manager(self.mock_si) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_profile_manager_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + type(self.mock_content).profileManager = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.get_profile_manager(self.mock_si) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') From c790107d17ba097dcdc71cfc21b6d6cf126665bd Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 13:42:21 -0400 Subject: [PATCH 06/47] Added salt.utils.pbm.get_placement_solver --- salt/utils/pbm.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index aec534111224..eb2cf2688735 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -93,3 +93,28 @@ def get_profile_manager(service_instance): log.exception(exc) raise VMwareRuntimeError(exc.msg) return profile_manager + + +def get_placement_solver(service_instance): + ''' + Returns a placement solver + + service_instance + Service instance to the host or vCenter + ''' + stub = salt.utils.vmware.get_new_service_instance_stub( + service_instance, ns='pbm/2.0', path='/pbm/sdk') + pbm_si = pbm.ServiceInstance('ServiceInstance', stub) + try: + profile_manager = pbm_si.RetrieveContent().placementSolver + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) + return profile_manager From 68f48d123ae51a66f61cefe844d39ce1c34af697 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 13:42:56 -0400 Subject: [PATCH 07/47] Added tests for salt.utils.pbm.get_placement_solver --- tests/unit/utils/test_pbm.py | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index 11256c9b32e2..8bdcbaa07529 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -103,3 +103,78 @@ def test_profile_manager_raises_runtime_fault(self): with self.assertRaises(VMwareRuntimeError) as excinfo: salt.utils.pbm.get_profile_manager(self.mock_si) self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetPlacementSolverTestCase(TestCase): + '''Tests for salt.utils.pbm.get_placement_solver''' + def setUp(self): + self.mock_si = MagicMock() + self.mock_stub = MagicMock() + self.mock_prof_mgr = MagicMock() + self.mock_content = MagicMock() + self.mock_pbm_si = MagicMock( + RetrieveContent=MagicMock(return_value=self.mock_content)) + type(self.mock_content).placementSolver = \ + PropertyMock(return_value=self.mock_prof_mgr) + patches = ( + ('salt.utils.vmware.get_new_service_instance_stub', + MagicMock(return_value=self.mock_stub)), + ('salt.utils.pbm.pbm.ServiceInstance', + MagicMock(return_value=self.mock_pbm_si))) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attr in ('mock_si', 'mock_stub', 'mock_content', + 'mock_pbm_si', 'mock_prof_mgr'): + delattr(self, attr) + + def test_get_new_service_stub(self): + mock_get_new_service_stub = MagicMock() + with patch('salt.utils.vmware.get_new_service_instance_stub', + mock_get_new_service_stub): + salt.utils.pbm.get_placement_solver(self.mock_si) + mock_get_new_service_stub.assert_called_once_with( + self.mock_si, ns='pbm/2.0', path='/pbm/sdk') + + def test_pbm_si(self): + mock_get_pbm_si = MagicMock() + with patch('salt.utils.pbm.pbm.ServiceInstance', + mock_get_pbm_si): + salt.utils.pbm.get_placement_solver(self.mock_si) + mock_get_pbm_si.assert_called_once_with('ServiceInstance', + self.mock_stub) + + def test_return_profile_manager(self): + ret = salt.utils.pbm.get_placement_solver(self.mock_si) + self.assertEqual(ret, self.mock_prof_mgr) + + def test_placement_solver_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + type(self.mock_content).placementSolver = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_placement_solver(self.mock_si) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_placement_solver_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + type(self.mock_content).placementSolver = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_placement_solver(self.mock_si) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_placement_solver_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + type(self.mock_content).placementSolver = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.get_placement_solver(self.mock_si) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') From eac509bab8bf1a6d8f9102b59f6e1daa3618984b Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 18:39:47 -0400 Subject: [PATCH 08/47] Added salt.utils.pbm.get_capability_definitions --- salt/utils/pbm.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index eb2cf2688735..5ca85ce4d991 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -118,3 +118,30 @@ def get_placement_solver(service_instance): log.exception(exc) raise VMwareRuntimeError(exc.msg) return profile_manager + + +def get_capability_definitions(profile_manager): + ''' + Returns a list of all capability definitions. + + profile_manager + Reference to the profile manager. + ''' + res_type = pbm.profile.ResourceType( + resourceType=pbm.profile.ResourceTypeEnum.STORAGE) + try: + cap_categories = profile_manager.FetchCapabilityMetadata(res_type) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) + cap_definitions = [] + for cat in cap_categories: + cap_definitions.extend(cat.capabilityMetadata) + return cap_definitions From e980407c54dea63ea35e642bb9ef2ba2d3bdc6a9 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 18:40:41 -0400 Subject: [PATCH 09/47] Added tests for salt.utils.pbm.get_capability_definitions --- tests/unit/utils/test_pbm.py | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index 8bdcbaa07529..d59ce1afdd8d 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -178,3 +178,74 @@ def test_placement_solver_raises_runtime_fault(self): with self.assertRaises(VMwareRuntimeError) as excinfo: salt.utils.pbm.get_placement_solver(self.mock_si) self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetCapabilityDefinitionsTestCase(TestCase): + '''Tests for salt.utils.pbm.get_capability_definitions''' + def setUp(self): + self.mock_res_type = MagicMock() + self.mock_cap_cats =[MagicMock(capabilityMetadata=['fake_cap_meta1', + 'fake_cap_meta2']), + MagicMock(capabilityMetadata=['fake_cap_meta3'])] + self.mock_prof_mgr = MagicMock( + FetchCapabilityMetadata=MagicMock(return_value=self.mock_cap_cats)) + patches = ( + ('salt.utils.pbm.pbm.profile.ResourceType', + MagicMock(return_value=self.mock_res_type)),) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attr in ('mock_res_type', 'mock_cap_cats', 'mock_prof_mgr'): + delattr(self, attr) + + def test_get_res_type(self): + mock_get_res_type = MagicMock() + with patch('salt.utils.pbm.pbm.profile.ResourceType', + mock_get_res_type): + salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) + mock_get_res_type.assert_called_once_with( + resourceType=pbm.profile.ResourceTypeEnum.STORAGE) + + def test_fetch_capabilities(self): + salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) + self.mock_prof_mgr.FetchCapabilityMetadata.assert_callend_once_with( + self.mock_res_type) + + def test_fetch_capabilities_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_prof_mgr.FetchCapabilityMetadata = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_fetch_capabilities_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + self.mock_prof_mgr.FetchCapabilityMetadata = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_fetch_capabilities_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + self.mock_prof_mgr.FetchCapabilityMetadata = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + def test_return_cap_definitions(self): + ret = salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) + self.assertEqual(ret, ['fake_cap_meta1', 'fake_cap_meta2', + 'fake_cap_meta3']) From f42de9c66b9e8df28dc10b1d32a197990ec1a849 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 18:41:41 -0400 Subject: [PATCH 10/47] Added salt.utils.pbm.get_policies_by_id --- salt/utils/pbm.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index 5ca85ce4d991..bf589b06c00c 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -145,3 +145,27 @@ def get_capability_definitions(profile_manager): for cat in cap_categories: cap_definitions.extend(cat.capabilityMetadata) return cap_definitions + + +def get_policies_by_id(profile_manager, policy_ids): + ''' + Returns a list of policies with the specified ids. + + profile_manager + Reference to the profile manager. + + policy_ids + List of policy ids to retrieve. + ''' + try: + return profile_manager.RetrieveContent(policy_ids) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) From d8e0cbde9ac679c7beb64ea91b892e717dc31523 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 18:42:29 -0400 Subject: [PATCH 11/47] Added tests for salt.utils.pbm.get_policies_by_id --- tests/unit/utils/test_pbm.py | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index d59ce1afdd8d..100a7313bf6c 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -249,3 +249,53 @@ def test_return_cap_definitions(self): ret = salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) self.assertEqual(ret, ['fake_cap_meta1', 'fake_cap_meta2', 'fake_cap_meta3']) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetPoliciesById(TestCase): + '''Tests for salt.utils.pbm.get_policies_by_id''' + def setUp(self): + self.policy_ids = MagicMock() + self.mock_policies = MagicMock() + self.mock_prof_mgr = MagicMock( + RetrieveContent=MagicMock(return_value=self.mock_policies)) + + def tearDown(self): + for attr in ('policy_ids', 'mock_policies', 'mock_prof_mgr'): + delattr(self, attr) + + def test_retrieve_policies(self): + salt.utils.pbm.get_policies_by_id(self.mock_prof_mgr, self.policy_ids) + self.mock_prof_mgr.RetrieveContent.assert_callend_once_with( + self.policy_ids) + + def test_retrieve_policies_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_prof_mgr.RetrieveContent = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_policies_by_id(self.mock_prof_mgr, self.policy_ids) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_retrieve_policies_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + self.mock_prof_mgr.RetrieveContent = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_policies_by_id(self.mock_prof_mgr, self.policy_ids) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_retrieve_policies_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + self.mock_prof_mgr.RetrieveContent = MagicMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.get_policies_by_id(self.mock_prof_mgr, self.policy_ids) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + def test_return_policies(self): + ret = salt.utils.pbm.get_policies_by_id(self.mock_prof_mgr, self.policy_ids) + self.assertEqual(ret, self.mock_policies) From df16bdb686446867c2f1e79c43bbfdc069232b0e Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 18:43:22 -0400 Subject: [PATCH 12/47] Added salt.utils.pbm.get_storage_policies --- salt/utils/pbm.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index bf589b06c00c..8bab7144352e 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -169,3 +169,42 @@ def get_policies_by_id(profile_manager, policy_ids): except vmodl.RuntimeFault as exc: log.exception(exc) raise VMwareRuntimeError(exc.msg) + + +def get_storage_policies(profile_manager, policy_names=[], + get_all_policies=False): + ''' + Returns a list of the storage policies, filtered by name. + + profile_manager + Reference to the profile manager. + + policy_names + List of policy names to filter by. + + get_all_policies + Flag specifying to return all policies, regardless of the specified + filter. + ''' + res_type = pbm.profile.ResourceType( + resourceType=pbm.profile.ResourceTypeEnum.STORAGE) + try: + policy_ids = profile_manager.QueryProfile(res_type) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) + log.trace('policy_ids = {0}'.format(policy_ids)) + # More policies are returned so we need to filter again + policies = [p for p in get_policies_by_id(profile_manager, policy_ids) + if p.resourceType.resourceType == + pbm.profile.ResourceTypeEnum.STORAGE] + if get_all_policies: + return policies + return [p for p in policies if p.name in policy_names] From 75764567c44002a3e71a8ca375a37b6c3dad3a09 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 18:44:07 -0400 Subject: [PATCH 13/47] Added tests for salt.utils.pbm.get_storage_policies --- tests/unit/utils/test_pbm.py | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index 100a7313bf6c..829f6c293ae4 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -299,3 +299,93 @@ def test_retrieve_policies_raises_runtime_fault(self): def test_return_policies(self): ret = salt.utils.pbm.get_policies_by_id(self.mock_prof_mgr, self.policy_ids) self.assertEqual(ret, self.mock_policies) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetStoragePoliciesTestCase(TestCase): + '''Tests for salt.utils.pbm.get_storage_policies''' + def setUp(self): + self.mock_res_type = MagicMock() + self.mock_policy_ids = MagicMock() + self.mock_prof_mgr = MagicMock( + QueryProfile=MagicMock(return_value=self.mock_policy_ids)) + # Policies + self.mock_policies=[] + for i in range(4): + mock_obj = MagicMock(resourceType=MagicMock( + resourceType=pbm.profile.ResourceTypeEnum.STORAGE)) + mock_obj.name = 'fake_policy{0}'.format(i) + self.mock_policies.append(mock_obj) + patches = ( + ('salt.utils.pbm.pbm.profile.ResourceType', + MagicMock(return_value=self.mock_res_type)), + ('salt.utils.pbm.get_policies_by_id', + MagicMock(return_value=self.mock_policies))) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attr in ('mock_res_type', 'mock_policy_ids', 'mock_policies', + 'mock_prof_mgr'): + delattr(self, attr) + + def test_get_res_type(self): + mock_get_res_type = MagicMock() + with patch('salt.utils.pbm.pbm.profile.ResourceType', + mock_get_res_type): + salt.utils.pbm.get_storage_policies(self.mock_prof_mgr) + mock_get_res_type.assert_called_once_with( + resourceType=pbm.profile.ResourceTypeEnum.STORAGE) + + def test_retrieve_policy_ids(self): + mock_retrieve_policy_ids = MagicMock(return_value=self.mock_policy_ids) + self.mock_prof_mgr.QueryProfile = mock_retrieve_policy_ids + salt.utils.pbm.get_storage_policies(self.mock_prof_mgr) + mock_retrieve_policy_ids.asser_called_once_with(self.mock_res_type) + + def test_retrieve_policy_ids_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_prof_mgr.QueryProfile = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_storage_policies(self.mock_prof_mgr) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_retrieve_policy_ids_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + self.mock_prof_mgr.QueryProfile = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_storage_policies(self.mock_prof_mgr) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_retrieve_policy_ids_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + self.mock_prof_mgr.QueryProfile = MagicMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.get_storage_policies(self.mock_prof_mgr) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + def test_get_policies_by_id(self): + mock_get_policies_by_id = MagicMock(return_value=self.mock_policies) + with patch('salt.utils.pbm.get_policies_by_id', + mock_get_policies_by_id): + salt.utils.pbm.get_storage_policies(self.mock_prof_mgr) + mock_get_policies_by_id.assert_called_once_with( + self.mock_prof_mgr, self.mock_policy_ids) + + def test_return_all_policies(self): + ret = salt.utils.pbm.get_storage_policies(self.mock_prof_mgr, + get_all_policies=True) + self.assertEqual(ret, self.mock_policies) + + def test_return_filtered_policies(self): + ret = salt.utils.pbm.get_storage_policies( + self.mock_prof_mgr, policy_names=['fake_policy1', 'fake_policy3']) + self.assertEqual(ret, [self.mock_policies[1], self.mock_policies[3]]) From d3744c80030a1d54c2978fa73fa7ed80eac76f35 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 19:44:48 -0400 Subject: [PATCH 14/47] Added salt.utils.pbm.create_storage_policy --- salt/utils/pbm.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index 8bab7144352e..eb45c96da28e 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -208,3 +208,27 @@ def get_storage_policies(profile_manager, policy_names=[], if get_all_policies: return policies return [p for p in policies if p.name in policy_names] + + +def create_storage_policy(profile_manager, policy_spec): + ''' + Creates a storage policy. + + profile_manager + Reference to the profile manager. + + policy_spec + Policy update spec. + ''' + try: + profile_manager.Create(policy_spec) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) From c80df65776c9caaafa7c8e900bf9b5b9705dfa10 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 19:46:47 -0400 Subject: [PATCH 15/47] Fixed tests for salt.utils.pbm.get_policies_by_id --- tests/unit/utils/test_pbm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index 829f6c293ae4..538448720dc7 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -253,7 +253,7 @@ def test_return_cap_definitions(self): @skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') -class GetPoliciesById(TestCase): +class GetPoliciesByIdTestCase(TestCase): '''Tests for salt.utils.pbm.get_policies_by_id''' def setUp(self): self.policy_ids = MagicMock() @@ -344,7 +344,7 @@ def test_retrieve_policy_ids(self): mock_retrieve_policy_ids = MagicMock(return_value=self.mock_policy_ids) self.mock_prof_mgr.QueryProfile = mock_retrieve_policy_ids salt.utils.pbm.get_storage_policies(self.mock_prof_mgr) - mock_retrieve_policy_ids.asser_called_once_with(self.mock_res_type) + mock_retrieve_policy_ids.assert_called_once_with(self.mock_res_type) def test_retrieve_policy_ids_raises_no_permissions(self): exc = vim.fault.NoPermission() From d43e3421350fb5ef81bb68780004dceda4d26ac8 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 19:47:36 -0400 Subject: [PATCH 16/47] Added tests for salt.utils.pbm.create_storage_policy --- tests/unit/utils/test_pbm.py | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index 538448720dc7..789d0c56d4d7 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -389,3 +389,51 @@ def test_return_filtered_policies(self): ret = salt.utils.pbm.get_storage_policies( self.mock_prof_mgr, policy_names=['fake_policy1', 'fake_policy3']) self.assertEqual(ret, [self.mock_policies[1], self.mock_policies[3]]) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class CreateStoragePolicyTestCase(TestCase): + '''Tests for salt.utils.pbm.create_storage_policy''' + def setUp(self): + self.mock_policy_spec = MagicMock() + self.mock_prof_mgr = MagicMock() + + def tearDown(self): + for attr in ('mock_policy_spec', 'mock_prof_mgr'): + delattr(self, attr) + + def test_create_policy(self): + salt.utils.pbm.create_storage_policy(self.mock_prof_mgr, + self.mock_policy_spec) + self.mock_prof_mgr.Create.assert_called_once_with( + self.mock_policy_spec) + + def test_create_policy_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_prof_mgr.Create = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.create_storage_policy(self.mock_prof_mgr, + self.mock_policy_spec) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_create_policy_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + self.mock_prof_mgr.Create = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.create_storage_policy(self.mock_prof_mgr, + self.mock_policy_spec) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_create_policy_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + self.mock_prof_mgr.Create = MagicMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.create_storage_policy(self.mock_prof_mgr, + self.mock_policy_spec) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') From 9c05f7c7341ee1b1de218c299d04be800e4e10d3 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 19:49:24 -0400 Subject: [PATCH 17/47] Added salt.utils.pbm.update_storage_policy --- salt/utils/pbm.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index eb45c96da28e..57d2f598d42b 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -232,3 +232,30 @@ def create_storage_policy(profile_manager, policy_spec): except vmodl.RuntimeFault as exc: log.exception(exc) raise VMwareRuntimeError(exc.msg) + + +def update_storage_policy(profile_manager, policy, policy_spec): + ''' + Updates a storage policy. + + profile_manager + Reference to the profile manager. + + policy + Reference to the policy to be updated. + + policy_spec + Policy update spec. + ''' + try: + profile_manager.Update(policy.profileId, policy_spec) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) From 79419702d934e53a0f621da3bb47328a9928ca62 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 19:50:14 -0400 Subject: [PATCH 18/47] Added tests for salt.utils.pbm.update_storage_policy --- tests/unit/utils/test_pbm.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index 789d0c56d4d7..f398f5a4ea44 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -437,3 +437,52 @@ def test_create_policy_raises_runtime_fault(self): salt.utils.pbm.create_storage_policy(self.mock_prof_mgr, self.mock_policy_spec) self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class UpdateStoragePolicyTestCase(TestCase): + '''Tests for salt.utils.pbm.update_storage_policy''' + def setUp(self): + self.mock_policy_spec = MagicMock() + self.mock_policy = MagicMock() + self.mock_prof_mgr = MagicMock() + + def tearDown(self): + for attr in ('mock_policy_spec', 'mock_policy', 'mock_prof_mgr'): + delattr(self, attr) + + def test_create_policy(self): + salt.utils.pbm.update_storage_policy( + self.mock_prof_mgr, self.mock_policy, self.mock_policy_spec) + self.mock_prof_mgr.Update.assert_called_once_with( + self.mock_policy.profileId, self.mock_policy_spec) + + def test_create_policy_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_prof_mgr.Update = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.update_storage_policy( + self.mock_prof_mgr, self.mock_policy, self.mock_policy_spec) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_create_policy_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + self.mock_prof_mgr.Update = MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.update_storage_policy( + self.mock_prof_mgr, self.mock_policy, self.mock_policy_spec) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_create_policy_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + self.mock_prof_mgr.Update = MagicMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.update_storage_policy( + self.mock_prof_mgr, self.mock_policy, self.mock_policy_spec) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') From 61c226c086e370e00546adf77f8e3c1039d7772c Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 19:51:07 -0400 Subject: [PATCH 19/47] Added salt.utils.pbm.get_default_storage_policy_of_datastore --- salt/utils/pbm.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index 57d2f598d42b..cb6474be852c 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -259,3 +259,36 @@ def update_storage_policy(profile_manager, policy, policy_spec): except vmodl.RuntimeFault as exc: log.exception(exc) raise VMwareRuntimeError(exc.msg) + + +def get_default_storage_policy_of_datastore(profile_manager, datastore): + ''' + Returns the default storage policy reference assigned to a datastore. + + profile_manager + Reference to the profile manager. + + datastore + Reference to the datastore. + ''' + # Retrieve all datastores visible + hub = pbm.placement.PlacementHub( + hubId=datastore._moId, hubType='Datastore') + log.trace('placement_hub = {0}'.format(hub)) + try: + policy_id = profile_manager.QueryDefaultRequirementProfile(hub) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) + policy_refs = get_policies_by_id(profile_manager, [policy_id]) + if not policy_refs: + raise VMwareObjectRetrievalError('Storage policy with id \'{0}\' was ' + 'not found'.format(policy_id)) + return policy_refs[0] From 5dbbac182d86bbd51cb22dcad89ae495a5730f57 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 19:51:38 -0400 Subject: [PATCH 20/47] Added tests for salt.utils.pbm.get_default_storage_policy_of_datastore --- tests/unit/utils/test_pbm.py | 106 ++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index f398f5a4ea44..b8803c475f08 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -16,7 +16,8 @@ PropertyMock # Import Salt libraries -from salt.exceptions import VMwareApiError, VMwareRuntimeError +from salt.exceptions import VMwareApiError, VMwareRuntimeError, \ + VMwareObjectRetrievalError import salt.utils.pbm try: @@ -486,3 +487,106 @@ def test_create_policy_raises_runtime_fault(self): salt.utils.pbm.update_storage_policy( self.mock_prof_mgr, self.mock_policy, self.mock_policy_spec) self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetDefaultStoragePolicyOfDatastoreTestCase(TestCase): + '''Tests for salt.utils.pbm.get_default_storage_policy_of_datastore''' + def setUp(self): + self.mock_ds = MagicMock(_moId='fake_ds_moid') + self.mock_hub = MagicMock() + self.mock_policy_id = 'fake_policy_id' + self.mock_prof_mgr = MagicMock( + QueryDefaultRequirementProfile=MagicMock( + return_value=self.mock_policy_id)) + self.mock_policy_refs = [MagicMock()] + patches = ( + ('salt.utils.pbm.pbm.placement.PlacementHub', + MagicMock(return_value=self.mock_hub)), + ('salt.utils.pbm.get_policies_by_id', + MagicMock(return_value=self.mock_policy_refs))) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attr in ('mock_ds', 'mock_hub', 'mock_policy_id', 'mock_prof_mgr', + 'mock_policy_refs'): + delattr(self, attr) + + def test_get_placement_hub(self): + mock_get_placement_hub = MagicMock() + with patch('salt.utils.pbm.pbm.placement.PlacementHub', + mock_get_placement_hub): + salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + mock_get_placement_hub.assert_called_once_with( + hubId='fake_ds_moid', hubType='Datastore') + + def test_query_default_requirement_profile(self): + mock_query_prof = MagicMock(return_value=self.mock_policy_id) + self.mock_prof_mgr.QueryDefaultRequirementProfile = \ + mock_query_prof + salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + mock_query_prof.assert_called_once_with(self.mock_hub) + + def test_query_default_requirement_profile_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_prof_mgr.QueryDefaultRequirementProfile = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_query_default_requirement_profile_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + self.mock_prof_mgr.QueryDefaultRequirementProfile = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_query_default_requirement_profile_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + self.mock_prof_mgr.QueryDefaultRequirementProfile = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + def test_get_policies_by_id(self): + mock_get_policies_by_id = MagicMock() + with patch('salt.utils.pbm.get_policies_by_id', + mock_get_policies_by_id): + salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + mock_get_policies_by_id.assert_called_once_with( + self.mock_prof_mgr, [self.mock_policy_id]) + + def test_no_policy_refs(self): + mock_get_policies_by_id = MagicMock() + with path('salt.utils.pbm.get_policies_by_id', + MagicMock(return_value=None)): + with self.assertRaises(VMwareObjectRetrievalError) as excinfo: + salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + self.assertEqual(excinfo.exception.strerror, + 'Storage policy with id \'fake_policy_id\' was not ' + 'found') + + def test_no_policy_refs(self): + mock_get_policies_by_id = MagicMock() + ret = salt.utils.pbm.get_default_storage_policy_of_datastore( + self.mock_prof_mgr, self.mock_ds) + self.assertEqual(ret, self.mock_policy_refs[0]) From 20fca4be441df1e0794f312a5e752c20534f6e0a Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 20:05:05 -0400 Subject: [PATCH 21/47] Added salt.utils.pbm.assign_default_storage_policy_to_datastore --- salt/utils/pbm.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index cb6474be852c..17b25acecaa6 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -292,3 +292,35 @@ def get_default_storage_policy_of_datastore(profile_manager, datastore): raise VMwareObjectRetrievalError('Storage policy with id \'{0}\' was ' 'not found'.format(policy_id)) return policy_refs[0] + + +def assign_default_storage_policy_to_datastore(profile_manager, policy, + datastore): + ''' + Assigns a storage policy as the default policy to a datastore. + + profile_manager + Reference to the profile manager. + + policy + Reference to the policy to assigned. + + datastore + Reference to the datastore. + ''' + placement_hub = pbm.placement.PlacementHub( + hubId=datastore._moId, hubType='Datastore') + log.trace('placement_hub = {0}'.format(placement_hub)) + try: + profile_manager.AssignDefaultRequirementProfile(policy.profileId, + [placement_hub]) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) From a3047ad3071c4c64d0e18035207c2bbf8d188519 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 19 Sep 2017 20:05:43 -0400 Subject: [PATCH 22/47] Added tests for salt.utils.pbm.assign_default_storage_policy_to_datastore --- tests/unit/utils/test_pbm.py | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index b8803c475f08..4e08229e261a 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -590,3 +590,75 @@ def test_no_policy_refs(self): ret = salt.utils.pbm.get_default_storage_policy_of_datastore( self.mock_prof_mgr, self.mock_ds) self.assertEqual(ret, self.mock_policy_refs[0]) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class AssignDefaultStoragePolicyToDatastoreTestCase(TestCase): + '''Tests for salt.utils.pbm.assign_default_storage_policy_to_datastore''' + def setUp(self): + self.mock_ds = MagicMock(_moId='fake_ds_moid') + self.mock_policy = MagicMock() + self.mock_hub = MagicMock() + self.mock_prof_mgr = MagicMock() + patches = ( + ('salt.utils.pbm.pbm.placement.PlacementHub', + MagicMock(return_value=self.mock_hub)),) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attr in ('mock_ds', 'mock_hub', 'mock_policy', 'mock_prof_mgr'): + delattr(self, attr) + + def test_get_placement_hub(self): + mock_get_placement_hub = MagicMock() + with patch('salt.utils.pbm.pbm.placement.PlacementHub', + mock_get_placement_hub): + salt.utils.pbm.assign_default_storage_policy_to_datastore( + self.mock_prof_mgr, self.mock_policy, self.mock_ds) + mock_get_placement_hub.assert_called_once_with( + hubId='fake_ds_moid', hubType='Datastore') + + def test_assign_default_requirement_profile(self): + mock_assign_prof = MagicMock() + self.mock_prof_mgr.AssignDefaultRequirementProfile = \ + mock_assign_prof + salt.utils.pbm.assign_default_storage_policy_to_datastore( + self.mock_prof_mgr, self.mock_policy, self.mock_ds) + mock_assign_prof.assert_called_once_with( + self.mock_policy.profileId, [self.mock_hub]) + + def test_assign_default_requirement_profile_raises_no_permissions(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_prof_mgr.AssignDefaultRequirementProfile = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.assign_default_storage_policy_to_datastore( + self.mock_prof_mgr, self.mock_policy, self.mock_ds) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_assign_default_requirement_profile_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + self.mock_prof_mgr.AssignDefaultRequirementProfile = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + salt.utils.pbm.assign_default_storage_policy_to_datastore( + self.mock_prof_mgr, self.mock_policy, self.mock_ds) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_assign_default_requirement_profile_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + self.mock_prof_mgr.AssignDefaultRequirementProfile = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + salt.utils.pbm.assign_default_storage_policy_to_datastore( + self.mock_prof_mgr, self.mock_policy, self.mock_ds) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') From 6da3ff5d933aa4631f7d8d5d9faaad582d30ebdb Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 10:03:36 -0400 Subject: [PATCH 23/47] Added salt.modules.vsphere._get_policy_dict that transforms a policy VMware object into a dict representation --- salt/modules/vsphere.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index bde7c9c98e15..84f9a7ace649 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -177,6 +177,7 @@ import salt.utils.path import salt.utils.vmware import salt.utils.vsan +import salt.utils.pbm from salt.exceptions import CommandExecutionError, VMwareSaltError, \ ArgumentValueError, InvalidConfigError, VMwareObjectRetrievalError, \ VMwareApiError, InvalidEntityError @@ -193,7 +194,7 @@ HAS_JSONSCHEMA = False try: - from pyVmomi import vim, vmodl, VmomiSupport + from pyVmomi import vim, vmodl, pbm, VmomiSupport HAS_PYVMOMI = True except ImportError: HAS_PYVMOMI = False @@ -4608,6 +4609,43 @@ def remove_dvportgroup(portgroup, dvs, service_instance=None): return True +def _get_policy_dict(policy): + '''Returns a dictionary representation of a policy''' + profile_dict = {'name': policy.name, + 'description': policy.description, + 'resource_type': policy.resourceType.resourceType} + subprofile_dicts = [] + if isinstance(policy, pbm.profile.CapabilityBasedProfile) and \ + isinstance(policy.constraints, + pbm.profile.SubProfileCapabilityConstraints): + + for subprofile in policy.constraints.subProfiles: + subprofile_dict = {'name': subprofile.name, + 'force_provision': subprofile.forceProvision} + cap_dicts = [] + for cap in subprofile.capability: + cap_dict = {'namespace': cap.id.namespace, + 'id': cap.id.id} + # We assume there is one constraint with one value set + val = cap.constraint[0].propertyInstance[0].value + if isinstance(val, pbm.capability.types.Range): + val_dict = {'type': 'range', + 'min': val.min, + 'max': val.max} + elif isinstance(val, pbm.capability.types.DiscreteSet): + val_dict = {'type': 'set', + 'values': val.values} + else: + val_dict = {'type': 'scalar', + 'value': val} + cap_dict['setting'] = val_dict + cap_dicts.append(cap_dict) + subprofile_dict['capabilities'] = cap_dicts + subprofile_dicts.append(subprofile_dict) + profile_dict['subprofiles'] = subprofile_dicts + return profile_dict + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From 6bb0111b327134d908e2061471d7732b362b3926 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 10:04:49 -0400 Subject: [PATCH 24/47] Added salt.modules.vsphere.list_storage_policies that retrieves dict representations of storage policies, filtered by name --- salt/modules/vsphere.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 84f9a7ace649..59181fd63499 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4646,6 +4646,36 @@ def _get_policy_dict(policy): return profile_dict +@depends(HAS_PYVMOMI) +@supports_proxies('esxdatacenter', 'vcenter') +@gets_service_instance_via_proxy +def list_storage_policies(policy_names=None, service_instance=None): + ''' + Returns a list of storage policies. + + policy_names + Names of policies to list. If None, all policies are listed. + Default is None. + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.list_storage_policies + + salt '*' vsphere.list_storage_policy policy_names=[policy_name] + ''' + profile_manager = salt.utils.pbm.get_profile_manager(service_instance) + if not policy_names: + policies = salt.utils.pbm.get_storage_policies(profile_manager, + get_all_policies=True) + else: + policies = salt.utils.pbm.get_storage_policies(profile_manager, + policy_names) + return [_get_policy_dict(p) for p in policies] + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From f9f84fde9ab8f45b183570665a315b31bd30d3e5 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 10:06:42 -0400 Subject: [PATCH 25/47] Added salt.modules.vsphere.list_default_vsan_policy that retrieves dict representation of the default storage policies --- salt/modules/vsphere.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 59181fd63499..96b2ac037e68 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4676,6 +4676,33 @@ def list_storage_policies(policy_names=None, service_instance=None): return [_get_policy_dict(p) for p in policies] +@depends(HAS_PYVMOMI) +@supports_proxies('esxdatacenter', 'vcenter') +@gets_service_instance_via_proxy +def list_default_vsan_policy(service_instance=None): + ''' + Returns the default vsan storage policy. + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.list_storage_policies + + salt '*' vsphere.list_storage_policy policy_names=[policy_name] + ''' + profile_manager = salt.utils.pbm.get_profile_manager(service_instance) + policies = salt.utils.pbm.get_storage_policies(profile_manager, + get_all_policies=True) + def_policies = [p for p in policies + if p.systemCreatedProfileType == 'VsanDefaultProfile'] + if not def_policies: + raise excs.VMwareObjectRetrievalError('Default VSAN policy was not ' + 'retrieved') + return _get_policy_dict(def_policies[0]) + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From 8275e5710681c3dcc585089014bd5887279ed728 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 10:10:48 -0400 Subject: [PATCH 26/47] Added salt.modules.vsphere._get_capability_definition_dict that transforms a VMware capability definition into a dict representation --- salt/modules/vsphere.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 96b2ac037e68..9655fd39fa85 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4703,6 +4703,17 @@ def list_default_vsan_policy(service_instance=None): return _get_policy_dict(def_policies[0]) +def _get_capability_definition_dict(cap_metadata): + # We assume each capability definition has one property with the same id + # as the capability so we display its type as belonging to the capability + # The object model permits multiple properties + return {'namespace': cap_metadata.id.namespace, + 'id': cap_metadata.id.id, + 'mandatory': cap_metadata.mandatory, + 'description': cap_metadata.summary.summary, + 'type': cap_metadata.propertyMetadata[0].type.typeName} + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From c88c207011821c7c2aad8d80a9eecb8be4befc4c Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 10:12:58 -0400 Subject: [PATCH 27/47] Added salt.modules.vsphere.list_capability_definitions that returns dict representations of VMware capability definition --- salt/modules/vsphere.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 9655fd39fa85..f92c3b6339cd 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4714,6 +4714,26 @@ def _get_capability_definition_dict(cap_metadata): 'type': cap_metadata.propertyMetadata[0].type.typeName} +@depends(HAS_PYVMOMI) +@supports_proxies('esxdatacenter', 'vcenter') +@gets_service_instance_via_proxy +def list_capability_definitions(service_instance=None): + ''' + Returns a list of the metadata of all capabilities in the vCenter. + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.list_capabilities + ''' + profile_manager = salt.utils.pbm.get_profile_manager(service_instance) + ret_list = [_get_capability_definition_dict(c) for c in + salt.utils.pbm.get_capability_definitions(profile_manager)] + return ret_list + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From ee2af6fc9c129539e39412c4b9ff79d6287d822c Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 12:26:14 -0400 Subject: [PATCH 28/47] Added salt.modules.vsphere._apply_policy_config that applies a storage dict representations values to a object --- salt/modules/vsphere.py | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index f92c3b6339cd..2a0000b8c5b5 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4734,6 +4734,55 @@ def list_capability_definitions(service_instance=None): return ret_list +def _apply_policy_config(policy_spec, policy_dict): + '''Applies a policy dictionary to a policy spec''' + log.trace('policy_dict = {0}'.format(policy_dict)) + if policy_dict.get('name'): + policy_spec.name = policy_dict['name'] + if policy_dict.get('description'): + policy_spec.description = policy_dict['description'] + if policy_dict.get('subprofiles'): + # Incremental changes to subprofiles and capabilities are not + # supported because they would complicate updates too much + # The whole configuration of all sub-profiles is expected and applied + policy_spec.constraints = pbm.profile.SubProfileCapabilityConstraints() + subprofiles = [] + for subprofile_dict in policy_dict['subprofiles']: + subprofile_spec = \ + pbm.profile.SubProfileCapabilityConstraints.SubProfile( + name=subprofile_dict['name']) + cap_specs = [] + if subprofile_dict.get('force_provision'): + subprofile_spec.forceProvision = \ + subprofile_dict['force_provision'] + for cap_dict in subprofile_dict['capabilities']: + prop_inst_spec = pbm.capability.PropertyInstance( + id=cap_dict['id'] + ) + setting_type = cap_dict['setting']['type'] + if setting_type == 'set': + prop_inst_spec.value = pbm.capability.types.DiscreteSet() + prop_inst_spec.value.values = cap_dict['setting']['values'] + elif setting_type == 'range': + prop_inst_spec.value = pbm.capability.types.Range() + prop_inst_spec.value.max = cap_dict['setting']['max'] + prop_inst_spec.value.min = cap_dict['setting']['min'] + elif setting_type == 'scalar': + prop_inst_spec.value = cap_dict['setting']['value'] + cap_spec = pbm.capability.CapabilityInstance( + id=pbm.capability.CapabilityMetadata.UniqueId( + id=cap_dict['id'], + namespace=cap_dict['namespace']), + constraint=[pbm.capability.ConstraintInstance( + propertyInstance=[prop_inst_spec])]) + cap_specs.append(cap_spec) + subprofile_spec.capability = cap_specs + subprofiles.append(subprofile_spec) + policy_spec.constraints.subProfiles = subprofiles + log.trace('updated policy_spec = {0}'.format(policy_spec)) + return policy_spec + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From a5ae51f6166267efdf1986b122dd978560e1db68 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 12:36:56 -0400 Subject: [PATCH 29/47] Added salt.modules.vsphere.create_storage_policy --- salt/modules/vsphere.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 2a0000b8c5b5..551ecedc7dca 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4783,6 +4783,47 @@ def _apply_policy_config(policy_spec, policy_dict): return policy_spec +@depends(HAS_PYVMOMI) +@supports_proxies('esxdatacenter', 'vcenter') +@gets_service_instance_via_proxy +def create_storage_policy(policy_name, policy_dict, service_instance=None): + ''' + Creates a storage policy. + + Supported capability types: scalar, set, range. + + policy_name + Name of the policy to create. + The value of the argument will override any existing name in + ``policy_dict``. + + policy_dict + Dictionary containing the changes to apply to the policy. + (exmaple in salt.states.pbm) + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.create_storage_policy policy_name='policy name' + policy_dict="$policy_dict" + ''' + log.trace('create storage policy \'{0}\', dict = {1}' + ''.format(policy_name, policy_dict)) + profile_manager = salt.utils.pbm.get_profile_manager(service_instance) + policy_create_spec = pbm.profile.CapabilityBasedProfileCreateSpec() + # Hardcode the storage profile resource type + policy_create_spec.resourceType = pbm.profile.ResourceType( + resourceType=pbm.profile.ResourceTypeEnum.STORAGE) + # Set name argument + policy_dict['name'] = policy_name + log.trace('Setting policy values in policy_update_spec') + _apply_policy_config(policy_create_spec, policy_dict) + salt.utils.pbm.create_storage_policy(profile_manager, policy_create_spec) + return {'create_storage_policy': True} + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From 41a65bf4140d31a84f395d533eaabbbbc8abb8c2 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 12:37:41 -0400 Subject: [PATCH 30/47] Added salt.modules.vsphere.update_storage_policy --- salt/modules/vsphere.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 551ecedc7dca..481df498a935 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4824,6 +4824,47 @@ def create_storage_policy(policy_name, policy_dict, service_instance=None): return {'create_storage_policy': True} +@depends(HAS_PYVMOMI) +@supports_proxies('esxdatacenter', 'vcenter') +@gets_service_instance_via_proxy +def update_storage_policy(policy, policy_dict, service_instance=None): + ''' + Updates a storage policy. + + Supported capability types: scalar, set, range. + + policy + Name of the policy to update. + + policy_dict + Dictionary containing the changes to apply to the policy. + (exmaple in salt.states.pbm) + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.update_storage_policy policy='policy name' + policy_dict="$policy_dict" + ''' + log.trace('updating storage policy, dict = {0}'.format(policy_dict)) + profile_manager = salt.utils.pbm.get_profile_manager(service_instance) + policies = salt.utils.pbm.get_storage_policies(profile_manager, [policy]) + if not policies: + raise excs.VMwareObjectRetrievalError('Policy \'{0}\' was not found' + ''.format(policy)) + policy_ref = policies[0] + policy_update_spec = pbm.profile.CapabilityBasedProfileUpdateSpec() + log.trace('Setting policy values in policy_update_spec') + for prop in ['description', 'constraints']: + setattr(policy_update_spec, prop, getattr(policy_ref, prop)) + _apply_policy_config(policy_update_spec, policy_dict) + salt.utils.pbm.update_storage_policy(profile_manager, policy_ref, + policy_update_spec) + return {'update_storage_policy': True} + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From 582919f5513ad25625cf9d81854a68129184dc1d Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 12:38:30 -0400 Subject: [PATCH 31/47] Added salt.modules.vsphere.list_default_storage_policy_of_datastore that lists the dict representation of the policy assigned by default to a datastore --- salt/modules/vsphere.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 481df498a935..cb6f6953c6fc 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4865,6 +4865,41 @@ def update_storage_policy(policy, policy_dict, service_instance=None): return {'update_storage_policy': True} +@depends(HAS_PYVMOMI) +@supports_proxies('esxcluster', 'esxdatacenter', 'vcenter') +@gets_service_instance_via_proxy +def list_default_storage_policy_of_datastore(datastore, service_instance=None): + ''' + Returns a list of datastores assign the the storage policies. + + datastore + Name of the datastore to assign. + The datastore needs to be visible to the VMware entity the proxy + points to. + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.list_default_storage_policy_of_datastore datastore=ds1 + ''' + log.trace('Listing the default storage policy of datastore \'{0}\'' + ''.format(datastore)) + # Find datastore + target_ref = _get_proxy_target(service_instance) + ds_refs = salt.utils.vmware.get_datastores(service_instance, target_ref, + datastore_names=[datastore]) + if not ds_refs: + raise excs.VMwareObjectRetrievalError('Datastore \'{0}\' was not ' + 'found'.format(datastore)) + profile_manager = salt.utils.pbm.get_profile_manager(service_instance) + policy = salt.utils.pbm.get_default_storage_policy_of_datastore( + profile_manager, ds_refs[0]) + return _get_policy_dict(policy) + + + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') @gets_service_instance_via_proxy From 0b2b79692a056498a5f1c87b0a1f1bb306e11627 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 18:10:40 -0400 Subject: [PATCH 32/47] Added salt.modules.vsphere.assign_default_storage_policy_to_datastore --- salt/modules/vsphere.py | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index cb6f6953c6fc..dce73ffa1a62 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4899,6 +4899,51 @@ def list_default_storage_policy_of_datastore(datastore, service_instance=None): return _get_policy_dict(policy) +@depends(HAS_PYVMOMI) +@supports_proxies('esxcluster', 'esxdatacenter', 'vcenter') +@gets_service_instance_via_proxy +def assign_default_storage_policy_to_datastore(policy, datastore, + service_instance=None): + ''' + Assigns a storage policy as the default policy to a datastore. + + policy + Name of the policy to assign. + + datastore + Name of the datastore to assign. + The datastore needs to be visible to the VMware entity the proxy + points to. + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.assign_storage_policy_to_datastore + policy='policy name' datastore=ds1 + ''' + log.trace('Assigning policy {0} to datastore {1}' + ''.format(policy, datastore)) + profile_manager = utils_pbm.get_profile_manager(service_instance) + # Find policy + policies = utils_pbm.get_storage_policies(profile_manager, [policy]) + if not policies: + raise excs.VMwareObjectRetrievalError('Policy \'{0}\' was not found' + ''.format(policy)) + policy_ref = policies[0] + # Find datastore + target_ref = _get_proxy_target(service_instance) + ds_refs = salt.utils.vmware.get_datastores(service_instance, target_ref, + datastore_names=[datastore]) + if not ds_refs: + raise excs.VMwareObjectRetrievalError('Datastore \'{0}\' was not ' + 'found'.format(datastore)) + ds_ref = ds_refs[0] + utils_pbm.assign_default_storage_policy_to_datastore(profile_manager, + policy_ref, ds_ref) + return {'assign_storage_policy_to_datastore': True} + @depends(HAS_PYVMOMI) @supports_proxies('esxdatacenter', 'esxcluster') From 507910b9560bcfe248f6ed4c6815d4625e62420f Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 20:10:04 -0400 Subject: [PATCH 33/47] Added VCenterProxySchema JSON schema that validates the vcenter proxy --- salt/config/schemas/vcenter.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/config/schemas/vcenter.py b/salt/config/schemas/vcenter.py index 4867923f27ac..1d76fb43a511 100644 --- a/salt/config/schemas/vcenter.py +++ b/salt/config/schemas/vcenter.py @@ -14,6 +14,8 @@ # Import Salt libs from salt.utils.schema import (Schema, + ArrayItem, + IntegerItem, StringItem) @@ -31,3 +33,25 @@ class VCenterEntitySchema(Schema): vcenter = StringItem(title='vCenter', description='Specifies the vcenter hostname', required=True) + + +class VCenterProxySchema(Schema): + ''' + Schema for the configuration for the proxy to connect to a VCenter. + ''' + title = 'VCenter Proxy Connection Schema' + description = 'Schema that describes the connection to a VCenter' + additional_properties = False + proxytype = StringItem(required=True, + enum=['vcenter']) + vcenter = StringItem(required=True, pattern=r'[^\s]+') + mechanism = StringItem(required=True, enum=['userpass', 'sspi']) + username = StringItem() + passwords = ArrayItem(min_items=1, + items=StringItem(), + unique_items=True) + + domain = StringItem() + principal = StringItem(default='host') + protocol = StringItem(default='https') + port = IntegerItem(minimum=1) From 176222b0cf262b0197930d91b9ac4fce07d5e687 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 20:27:19 -0400 Subject: [PATCH 34/47] Added vcenter proxy --- salt/proxy/vcenter.py | 338 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100644 salt/proxy/vcenter.py diff --git a/salt/proxy/vcenter.py b/salt/proxy/vcenter.py new file mode 100644 index 000000000000..7b9c9f95e304 --- /dev/null +++ b/salt/proxy/vcenter.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- +''' +Proxy Minion interface module for managing VMWare vCenters. + +:codeauthor: :email:`Rod McKenzie (roderick.mckenzie@morganstanley.com)` +:codeauthor: :email:`Alexandru Bleotu (alexandru.bleotu@morganstanley.com)` + +Dependencies +============ + +- pyVmomi Python Module + +pyVmomi +------- + +PyVmomi can be installed via pip: + +.. code-block:: bash + + pip install pyVmomi + +.. note:: + + Version 6.0 of pyVmomi has some problems with SSL error handling on certain + versions of Python. If using version 6.0 of pyVmomi, Python 2.6, + Python 2.7.9, or newer must be present. This is due to an upstream dependency + in pyVmomi 6.0 that is not supported in Python versions 2.7 to 2.7.8. If the + version of Python is not in the supported range, you will need to install an + earlier version of pyVmomi. See `Issue #29537`_ for more information. + +.. _Issue #29537: https://github.com/saltstack/salt/issues/29537 + +Based on the note above, to install an earlier version of pyVmomi than the +version currently listed in PyPi, run the following: + +.. code-block:: bash + + pip install pyVmomi==5.5.0.2014.1.1 + +The 5.5.0.2014.1.1 is a known stable version that this original ESXi State +Module was developed against. + + +Configuration +============= +To use this proxy module, please use on of the following configurations: + + +.. code-block:: yaml + + proxy: + proxytype: vcenter + vcenter: + username: + mechanism: userpass + passwords: + - first_password + - second_password + - third_password + + proxy: + proxytype: vcenter + vcenter: + username: + domain: + mechanism: sspi + principal: + +proxytype +^^^^^^^^^ +The ``proxytype`` key and value pair is critical, as it tells Salt which +interface to load from the ``proxy`` directory in Salt's install hierarchy, +or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your +own proxy module, for example). To use this Proxy Module, set this to +``vcenter``. + +vcenter +^^^^^^^ +The location of the VMware vCenter server (host of ip). Required + +username +^^^^^^^^ +The username used to login to the vcenter, such as ``root``. +Required only for userpass. + +mechanism +^^^^^^^^ +The mechanism used to connect to the vCenter server. Supported values are +``userpass`` and ``sspi``. Required. + +passwords +^^^^^^^^^ +A list of passwords to be used to try and login to the vCenter server. At least +one password in this list is required if mechanism is ``userpass`` + +The proxy integration will try the passwords listed in order. + +domain +^^^^^^ +User domain. Required if mechanism is ``sspi`` + +principal +^^^^^^^^ +Kerberos principal. Rquired if mechanism is ``sspi`` + +protocol +^^^^^^^^ +If the vCenter is not using the default protocol, set this value to an +alternate protocol. Default is ``https``. + +port +^^^^ +If the ESXi host is not using the default port, set this value to an +alternate port. Default is ``443``. + + +Salt Proxy +---------- + +After your pillar is in place, you can test the proxy. The proxy can run on +any machine that has network connectivity to your Salt Master and to the +vCenter server in the pillar. SaltStack recommends that the machine running the +salt-proxy process also run a regular minion, though it is not strictly +necessary. + +On the machine that will run the proxy, make sure there is an ``/etc/salt/proxy`` +file with at least the following in it: + +.. code-block:: yaml + + master: + +You can then start the salt-proxy process with: + +.. code-block:: bash + + salt-proxy --proxyid + +You may want to add ``-l debug`` to run the above in the foreground in +debug mode just to make sure everything is OK. + +Next, accept the key for the proxy on your salt-master, just like you +would for a regular minion: + +.. code-block:: bash + + salt-key -a + +You can confirm that the pillar data is in place for the proxy: + +.. code-block:: bash + + salt pillar.items + +And now you should be able to ping the ESXi host to make sure it is +responding: + +.. code-block:: bash + + salt test.ping + +At this point you can execute one-off commands against the vcenter. For +example, you can get if the proxy can actually connect to the vCenter: + +.. code-block:: bash + + salt vsphere.test_vcenter_connection + +Note that you don't need to provide credentials or an ip/hostname. Salt +knows to use the credentials you stored in Pillar. + +It's important to understand how this particular proxy works. +:mod:`Salt.modules.vsphere ` is a +standard Salt execution module. + + If you pull up the docs for it you'll see +that almost every function in the module takes credentials and a targets either +a vcenter or a host. When credentials and a host aren't passed, Salt runs commands +through ``pyVmomi`` against the local machine. If you wanted, you could run +functions from this module on any host where an appropriate version of +``pyVmomi`` is installed, and that host would reach out over the network +and communicate with the ESXi host. +''' + +# Import Python Libs +from __future__ import absolute_import +import logging +import os + +# Import Salt Libs +import salt.exceptions +from salt.config.schemas.vcenter import VCenterProxySchema +from salt.utils.dictupdate import merge + +# This must be present or the Salt loader won't load this module. +__proxyenabled__ = ['vcenter'] + +# External libraries +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + +# Variables are scoped to this module so we can have persistent data +# across calls to fns in here. +DETAILS = {} + + +# Set up logging +log = logging.getLogger(__name__) +# Define the module's virtual name +__virtualname__ = 'vcenter' + + +def __virtual__(): + ''' + Only load if the vsphere execution module is available. + ''' + if HAS_JSONSCHEMA: + return __virtualname__ + + return False, 'The vcenter proxy module did not load.' + + +def init(opts): + ''' + This function gets called when the proxy starts up. + For login the protocol and port are cached. + ''' + log.info('Initting vcenter proxy module in process {0}' + ''.format(os.getpid())) + log.trace('VCenter Proxy Validating vcenter proxy input') + schema = VCenterProxySchema.serialize() + log.trace('schema = {}'.format(schema)) + proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {})) + log.trace('proxy_conf = {0}'.format(proxy_conf)) + try: + jsonschema.validate(proxy_conf, schema) + except jsonschema.exceptions.ValidationError as exc: + raise salt.exceptions.InvalidConfigError(exc) + + # Save mandatory fields in cache + for key in ('vcenter', 'mechanism'): + DETAILS[key] = proxy_conf[key] + + # Additional validation + if DETAILS['mechanism'] == 'userpass': + if 'username' not in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'userpass\' , but no ' + '\'username\' key found in proxy config') + if not 'passwords' in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'userpass\' , but no ' + '\'passwords\' key found in proxy config') + for key in ('username', 'passwords'): + DETAILS[key] = proxy_conf[key] + else: + if not 'domain' in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'sspi\' , but no ' + '\'domain\' key found in proxy config') + if not 'principal' in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'sspi\' , but no ' + '\'principal\' key found in proxy config') + for key in ('domain', 'principal'): + DETAILS[key] = proxy_conf[key] + + # Save optional + DETAILS['protocol'] = proxy_conf.get('protocol') + DETAILS['port'] = proxy_conf.get('port') + + # Test connection + if DETAILS['mechanism'] == 'userpass': + # Get the correct login details + log.info('Retrieving credentials and testing vCenter connection for ' + 'mehchanism \'userpass\'') + try: + username, password = find_credentials() + DETAILS['password'] = password + except salt.exceptions.SaltSystemExit as err: + log.critical('Error: {0}'.format(err)) + return False + return True + + +def ping(): + ''' + Returns True. + + CLI Example: + + .. code-block:: bash + + salt vcenter test.ping + ''' + return True + + +def shutdown(): + ''' + Shutdown the connection to the proxy device. For this proxy, + shutdown is a no-op. + ''' + log.debug('VCenter proxy shutdown() called...') + + +def find_credentials(): + ''' + Cycle through all the possible credentials and return the first one that + works. + ''' + + # if the username and password were already found don't fo though the + # connection process again + if 'username' in DETAILS and 'password' in DETAILS: + return DETAILS['username'], DETAILS['password'] + + passwords = __pillar__['proxy']['passwords'] + for password in passwords: + DETAILS['password'] = password + if not __salt__['vsphere.test_vcenter_connection'](): + # We are unable to authenticate + continue + # If we have data returned from above, we've successfully authenticated. + return DETAILS['username'], password + # We've reached the end of the list without successfully authenticating. + raise salt.exceptions.VMwareConnectionError('Cannot complete login due to ' + 'incorrect credentials.') + + +def get_details(): + ''' + Function that returns the cached details + ''' + return DETAILS From 483fa0d8382ef1a10a656afe41bde10964b373c2 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 20:36:57 -0400 Subject: [PATCH 35/47] Added salt.modules.vcenter shim execution module between the proxy and other execution modules --- salt/modules/vcenter.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 salt/modules/vcenter.py diff --git a/salt/modules/vcenter.py b/salt/modules/vcenter.py new file mode 100644 index 000000000000..bac3c674b49a --- /dev/null +++ b/salt/modules/vcenter.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +''' +Module used to access the vcenter proxy connection methods +''' +from __future__ import absolute_import + +# Import python libs +import logging +import salt.utils + + +log = logging.getLogger(__name__) + +__proxyenabled__ = ['vcenter'] +# Define the module's virtual name +__virtualname__ = 'vcenter' + + +def __virtual__(): + ''' + Only work on proxy + ''' + if salt.utils.is_proxy(): + return __virtualname__ + return False + + +def get_details(): + return __proxy__['vcenter.get_details']() From 94929d541520456583bb549aab6d98b9b84c9142 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 20:38:14 -0400 Subject: [PATCH 36/47] Added support for vcenter proxy in salt.modules.vsphere --- salt/modules/vsphere.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index dce73ffa1a62..d4421ce1de21 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -208,7 +208,7 @@ log = logging.getLogger(__name__) __virtualname__ = 'vsphere' -__proxyenabled__ = ['esxi', 'esxcluster', 'esxdatacenter'] +__proxyenabled__ = ['esxi', 'esxcluster', 'esxdatacenter', 'vcenter'] def __virtual__(): @@ -255,6 +255,8 @@ def _get_proxy_connection_details(): details = __salt__['esxcluster.get_details']() elif proxytype == 'esxdatacenter': details = __salt__['esxdatacenter.get_details']() + elif proxytype == 'vcenter': + details = __salt__['vcenter.get_details']() else: raise CommandExecutionError('\'{0}\' proxy is not supported' ''.format(proxytype)) @@ -380,7 +382,7 @@ def _gets_service_instance_via_proxy(*args, **kwargs): @depends(HAS_PYVMOMI) -@supports_proxies('esxi', 'esxcluster', 'esxdatacenter') +@supports_proxies('esxi', 'esxcluster', 'esxdatacenter', 'vcenter') def get_service_instance_via_proxy(service_instance=None): ''' Returns a service instance to the proxied endpoint (vCenter/ESXi host). @@ -400,7 +402,7 @@ def get_service_instance_via_proxy(service_instance=None): @depends(HAS_PYVMOMI) -@supports_proxies('esxi', 'esxcluster', 'esxdatacenter') +@supports_proxies('esxi', 'esxcluster', 'esxdatacenter', 'vcenter') def disconnect(service_instance): ''' Disconnects from a vCenter or ESXi host @@ -1935,7 +1937,7 @@ def get_vsan_eligible_disks(host, username, password, protocol=None, port=None, @depends(HAS_PYVMOMI) -@supports_proxies('esxi', 'esxcluster', 'esxdatacenter') +@supports_proxies('esxi', 'esxcluster', 'esxdatacenter', 'vcenter') @gets_service_instance_via_proxy def test_vcenter_connection(service_instance=None): ''' @@ -4946,7 +4948,7 @@ def assign_default_storage_policy_to_datastore(policy, datastore, @depends(HAS_PYVMOMI) -@supports_proxies('esxdatacenter', 'esxcluster') +@supports_proxies('esxdatacenter', 'esxcluster', 'vcenter') @gets_service_instance_via_proxy def list_datacenters_via_proxy(datacenter_names=None, service_instance=None): ''' @@ -4984,7 +4986,7 @@ def list_datacenters_via_proxy(datacenter_names=None, service_instance=None): @depends(HAS_PYVMOMI) -@supports_proxies('esxdatacenter') +@supports_proxies('esxdatacenter', 'vcenter') @gets_service_instance_via_proxy def create_datacenter(datacenter_name, service_instance=None): ''' @@ -6439,7 +6441,7 @@ def add_host_to_dvs(host, username, password, vmknic_name, vmnic_name, @depends(HAS_PYVMOMI) -@supports_proxies('esxcluster', 'esxdatacenter') +@supports_proxies('esxcluster', 'esxdatacenter', 'vcenter') def _get_proxy_target(service_instance): ''' Returns the target object of a proxy. @@ -6467,6 +6469,9 @@ def _get_proxy_target(service_instance): reference = salt.utils.vmware.get_datacenter(service_instance, datacenter) + elif proxy_type == 'vcenter': + # vcenter proxy - the target is the root folder + reference = salt.utils.vmware.get_root_folder(service_instance) log.trace('reference = {0}'.format(reference)) return reference From 58445e927b8295bcd74f4047b619565458d0aeaa Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 21:00:39 -0400 Subject: [PATCH 37/47] Updated all vsphere tests to support the vcenter proxy --- tests/unit/modules/test_vsphere.py | 47 +++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/tests/unit/modules/test_vsphere.py b/tests/unit/modules/test_vsphere.py index 56669b900e80..9ebad773631c 100644 --- a/tests/unit/modules/test_vsphere.py +++ b/tests/unit/modules/test_vsphere.py @@ -639,6 +639,15 @@ def setUp(self): 'mechanism': 'fake_mechanism', 'principal': 'fake_principal', 'domain': 'fake_domain'} + self.vcenter_details = {'vcenter': 'fake_vcenter', + 'username': 'fake_username', + 'password': 'fake_password', + 'protocol': 'fake_protocol', + 'port': 'fake_port', + 'mechanism': 'fake_mechanism', + 'principal': 'fake_principal', + 'domain': 'fake_domain'} + def tearDown(self): for attrname in ('esxi_host_details', 'esxi_vcenter_details', @@ -693,6 +702,17 @@ def test_esxi_proxy_vcenter_details(self): 'fake_protocol', 'fake_port', 'fake_mechanism', 'fake_principal', 'fake_domain'), ret) + def test_vcenter_proxy_details(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='vcenter')): + with patch.dict(vsphere.__salt__, + {'vcenter.get_details': MagicMock( + return_value=self.vcenter_details)}): + ret = vsphere._get_proxy_connection_details() + self.assertEqual(('fake_vcenter', 'fake_username', 'fake_password', + 'fake_protocol', 'fake_port', 'fake_mechanism', + 'fake_principal', 'fake_domain'), ret) + def test_unsupported_proxy_details(self): with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value='unsupported')): @@ -890,7 +910,7 @@ def setup_loader_modules(self): } def test_supported_proxies(self): - supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter'] + supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter', 'vcenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -933,7 +953,7 @@ def setup_loader_modules(self): } def test_supported_proxies(self): - supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter'] + supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter', 'vcenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -974,7 +994,7 @@ def setup_loader_modules(self): } def test_supported_proxies(self): - supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter'] + supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter', 'vcenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -1049,7 +1069,7 @@ def setup_loader_modules(self): } def test_supported_proxies(self): - supported_proxies = ['esxcluster', 'esxdatacenter'] + supported_proxies = ['esxcluster', 'esxdatacenter', 'vcenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -1127,7 +1147,7 @@ def setup_loader_modules(self): } def test_supported_proxies(self): - supported_proxies = ['esxdatacenter'] + supported_proxies = ['esxdatacenter', 'vcenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -1339,12 +1359,15 @@ def setup_loader_modules(self): def setUp(self): attrs = (('mock_si', MagicMock()), ('mock_dc', MagicMock()), - ('mock_cl', MagicMock())) + ('mock_cl', MagicMock()), + ('mock_root', MagicMock())) for attr, mock_obj in attrs: setattr(self, attr, mock_obj) self.addCleanup(delattr, self, attr) attrs = (('mock_get_datacenter', MagicMock(return_value=self.mock_dc)), - ('mock_get_cluster', MagicMock(return_value=self.mock_cl))) + ('mock_get_cluster', MagicMock(return_value=self.mock_cl)), + ('mock_get_root_folder', + MagicMock(return_value=self.mock_root))) for attr, mock_obj in attrs: setattr(self, attr, mock_obj) self.addCleanup(delattr, self, attr) @@ -1360,7 +1383,8 @@ def setUp(self): MagicMock(return_value=(None, None, None, None, None, None, None, None, 'datacenter'))), ('salt.utils.vmware.get_datacenter', self.mock_get_datacenter), - ('salt.utils.vmware.get_cluster', self.mock_get_cluster)) + ('salt.utils.vmware.get_cluster', self.mock_get_cluster), + ('salt.utils.vmware.get_root_folder', self.mock_get_root_folder)) for module, mock_obj in patches: patcher = patch(module, mock_obj) patcher.start() @@ -1409,3 +1433,10 @@ def test_esxdatacenter_proxy_return(self): MagicMock(return_value='esxdatacenter')): ret = vsphere._get_proxy_target(self.mock_si) self.assertEqual(ret, self.mock_dc) + + def test_vcenter_proxy_return(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='vcenter')): + ret = vsphere._get_proxy_target(self.mock_si) + self.mock_get_root_folder.assert_called_once_with(self.mock_si) + self.assertEqual(ret, self.mock_root) From da39e7ce842d969032cd3184dde1024c450efdab Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 21:05:53 -0400 Subject: [PATCH 38/47] Comments, imports, init function in salt.states.pbm --- salt/states/pbm.py | 134 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 salt/states/pbm.py diff --git a/salt/states/pbm.py b/salt/states/pbm.py new file mode 100644 index 000000000000..3026368f4bc3 --- /dev/null +++ b/salt/states/pbm.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +''' +Manages VMware storage policies +(called pbm because the vCenter endpoint is /pbm) + +Examples +======== + +Storage policy +-------------- + +.. code-block:: python + +{ + "name": "salt_storage_policy" + "description": "Managed by Salt. Random capability values.", + "resource_type": "STORAGE", + "subprofiles": [ + { + "capabilities": [ + { + "setting": { + "type": "scalar", + "value": 2 + }, + "namespace": "VSAN", + "id": "hostFailuresToTolerate" + }, + { + "setting": { + "type": "scalar", + "value": 2 + }, + "namespace": "VSAN", + "id": "stripeWidth" + }, + { + "setting": { + "type": "scalar", + "value": true + }, + "namespace": "VSAN", + "id": "forceProvisioning" + }, + { + "setting": { + "type": "scalar", + "value": 50 + }, + "namespace": "VSAN", + "id": "proportionalCapacity" + }, + { + "setting": { + "type": "scalar", + "value": 0 + }, + "namespace": "VSAN", + "id": "cacheReservation" + } + ], + "name": "Rule-Set 1: VSAN", + "force_provision": null + } + ], +} + +Dependencies +============ + + +- pyVmomi Python Module + + +pyVmomi +------- + +PyVmomi can be installed via pip: + +.. code-block:: bash + + pip install pyVmomi + +.. note:: + + Version 6.0 of pyVmomi has some problems with SSL error handling on certain + versions of Python. If using version 6.0 of pyVmomi, Python 2.6, + Python 2.7.9, or newer must be present. This is due to an upstream dependency + in pyVmomi 6.0 that is not supported in Python versions 2.7 to 2.7.8. If the + version of Python is not in the supported range, you will need to install an + earlier version of pyVmomi. See `Issue #29537`_ for more information. + +.. _Issue #29537: https://github.com/saltstack/salt/issues/29537 +''' + +# Import Python Libs +from __future__ import absolute_import +import sys +import logging +import json +import time +import copy + +# Import Salt Libs +from salt.exceptions import CommandExecutionError, ArgumentValueError +import salt.modules.vsphere as vsphere +from salt.utils import is_proxy +from salt.utils.dictdiffer import recursive_diff +from salt.utils.listdiffer import list_diff + +# External libraries +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + +# Get Logging Started +log = logging.getLogger(__name__) +# TODO change with vcenter +ALLOWED_PROXY_TYPES = ['esxcluster', 'vcenter'] +LOGIN_DETAILS = {} + +def __virtual__(): + if HAS_JSONSCHEMA: + return True + return False + + +def mod_init(low): + ''' + Init function + ''' + return True From 9f96c1fcc091452c844165d8b1cd10cc4bdc1914 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 21:07:19 -0400 Subject: [PATCH 39/47] Added salt.states.pbm.default_vsan_policy_configured state that configures the default storage policy --- salt/states/pbm.py | 141 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/salt/states/pbm.py b/salt/states/pbm.py index 3026368f4bc3..a30eba6456b8 100644 --- a/salt/states/pbm.py +++ b/salt/states/pbm.py @@ -132,3 +132,144 @@ def mod_init(low): Init function ''' return True + + +def default_vsan_policy_configured(name, policy): + ''' + Configures the default VSAN policy on a vCenter. + The state assumes there is only one default VSAN policy on a vCenter. + + policy + Dict representation of a policy + ''' + # TODO Refactor when recurse_differ supports list_differ + # It's going to make the whole thing much easier + policy_copy = copy.deepcopy(policy) + proxy_type = __salt__['vsphere.get_proxy_type']() + log.trace('proxy_type = {0}'.format(proxy_type)) + # All allowed proxies have a shim execution module with the same + # name which implementes a get_details function + # All allowed proxies have a vcenter detail + vcenter = __salt__['{0}.get_details'.format(proxy_type)]()['vcenter'] + log.info('Running {0} on vCenter ' + '\'{1}\''.format(name, vcenter)) + log.trace('policy = {0}'.format(policy)) + changes_required = False + ret = {'name': name, 'changes': {}, 'result': None, 'comment': None, + 'pchanges': {}} + comments = [] + changes = {} + changes_required = False + si = None + + try: + #TODO policy schema validation + si = __salt__['vsphere.get_service_instance_via_proxy']() + current_policy = __salt__['vsphere.list_default_vsan_policy'](si) + log.trace('current_policy = {0}'.format(current_policy)) + # Building all diffs between the current and expected policy + # XXX We simplify the comparison by assuming we have at most 1 + # sub_profile + if policy.get('subprofiles'): + if len(policy['subprofiles']) > 1: + raise ArgumentValueError('Multiple sub_profiles ({0}) are not ' + 'supported in the input policy') + subprofile = policy['subprofiles'][0] + current_subprofile = current_policy['subprofiles'][0] + capabilities_differ = list_diff(current_subprofile['capabilities'], + subprofile.get('capabilities', []), + key='id') + del policy['subprofiles'] + if subprofile.get('capabilities'): + del subprofile['capabilities'] + del current_subprofile['capabilities'] + # Get the subprofile diffs without the capability keys + subprofile_differ = recursive_diff(current_subprofile, + dict(subprofile)) + + del current_policy['subprofiles'] + policy_differ = recursive_diff(current_policy, policy) + if policy_differ.diffs or capabilities_differ.diffs or \ + subprofile_differ.diffs: + + if 'name' in policy_differ.new_values or \ + 'description' in policy_differ.new_values: + + raise ArgumentValueError( + '\'name\' and \'description\' of the default VSAN policy ' + 'cannot be updated') + changes_required = True + if __opts__['test']: + str_changes = [] + if policy_differ.diffs: + str_changes.extend([change for change in + policy_differ.changes_str.split('\n')]) + if subprofile_differ.diffs or capabilities_differ.diffs: + str_changes.append('subprofiles:') + if subprofile_differ.diffs: + str_changes.extend( + [' {0}'.format(change) for change in + subprofile_differ.changes_str.split('\n')]) + if capabilities_differ.diffs: + str_changes.append(' capabilities:') + str_changes.extend( + [' {0}'.format(change) for change in + capabilities_differ.changes_str2.split('\n')]) + comments.append( + 'State {0} will update the default VSAN policy on ' + 'vCenter \'{1}\':\n{2}' + ''.format(name, vcenter, '\n'.join(str_changes))) + else: + __salt__['vsphere.update_storage_policy']( + policy=current_policy['name'], + policy_dict=policy_copy, + service_instance=si) + comments.append('Updated the default VSAN policy in vCenter ' + '\'{0}\''.format(vcenter)) + log.info(comments[-1]) + + new_values = policy_differ.new_values + new_values['subprofiles'] = [subprofile_differ.new_values] + new_values['subprofiles'][0]['capabilities'] = \ + capabilities_differ.new_values + if not new_values['subprofiles'][0]['capabilities']: + del new_values['subprofiles'][0]['capabilities'] + if not new_values['subprofiles'][0]: + del new_values['subprofiles'] + old_values = policy_differ.old_values + old_values['subprofiles'] = [subprofile_differ.old_values] + old_values['subprofiles'][0]['capabilities'] = \ + capabilities_differ.old_values + if not old_values['subprofiles'][0]['capabilities']: + del old_values['subprofiles'][0]['capabilities'] + if not old_values['subprofiles'][0]: + del old_values['subprofiles'] + changes.update({'default_vsan_policy': + {'new': new_values, + 'old': old_values}}) + log.trace(changes) + __salt__['vsphere.disconnect'](si) + except CommandExecutionError as exc: + log.error('Error: {}'.format(exc)) + if si: + __salt__['vsphere.disconnect'](si) + if not __opts__['test']: + ret['result'] = False + ret.update({'comment': exc.strerror, + 'result': False if not __opts__['test'] else None}) + return ret + if not changes_required: + # We have no changes + ret.update({'comment': ('Default VSAN policy in vCenter ' + '\'{0}\' is correctly configured. ' + 'Nothing to be done.'.format(vcenter)), + 'result': True}) + else: + ret.update({'comment': '\n'.join(comments)}) + if __opts__['test']: + ret.update({'pchanges': changes, + 'result': None}) + else: + ret.update({'changes': changes, + 'result': True}) + return ret From bb52e0d3318d1026061a1cc2350654ad86bdfb3e Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 21:08:42 -0400 Subject: [PATCH 40/47] Added salt.states.pbm.storage_policies_configured state that creates/configures storage policies --- salt/states/pbm.py | 164 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/salt/states/pbm.py b/salt/states/pbm.py index a30eba6456b8..54833151954b 100644 --- a/salt/states/pbm.py +++ b/salt/states/pbm.py @@ -273,3 +273,167 @@ def default_vsan_policy_configured(name, policy): ret.update({'changes': changes, 'result': True}) return ret + + +def storage_policies_configured(name, policies): + ''' + Configures storage policies on a vCenter. + + policies + List of dict representation of the required storage policies + ''' + comments = [] + changes = [] + changes_required = False + ret = {'name': name, 'changes': {}, 'result': None, 'comment': None, + 'pchanges': {}} + log.trace('policies = {0}'.format(policies)) + si = None + try: + proxy_type = __salt__['vsphere.get_proxy_type']() + log.trace('proxy_type = {0}'.format(proxy_type)) + # All allowed proxies have a shim execution module with the same + # name which implementes a get_details function + # All allowed proxies have a vcenter detail + vcenter = __salt__['{0}.get_details'.format(proxy_type)]()['vcenter'] + log.info('Running state \'{0}\' on vCenter ' + '\'{0}\''.format(name, vcenter)) + si = __salt__['vsphere.get_service_instance_via_proxy']() + current_policies = __salt__['vsphere.list_storage_policies']( + policy_names=[policy['name'] for policy in policies], + service_instance=si) + log.trace('current_policies = {0}'.format(current_policies)) + # TODO Refactor when recurse_differ supports list_differ + # It's going to make the whole thing much easier + for policy in policies: + policy_copy = copy.deepcopy(policy) + filtered_policies = [p for p in current_policies + if p['name'] == policy['name']] + current_policy = filtered_policies[0] \ + if filtered_policies else None + + if not current_policy: + changes_required = True + if __opts__['test']: + comments.append('State {0} will create the storage policy ' + '\'{1}\' on vCenter \'{2}\'' + ''.format(name, policy['name'], vcenter)) + else: + __salt__['vsphere.create_storage_policy']( + policy['name'], policy, service_instance=si) + comments.append('Created storage policy \'{0}\' on ' + 'vCenter \'{1}\''.format(policy['name'], + vcenter)) + changes.append({'new': policy, 'old': None}) + log.trace(comments[-1]) + # Continue with next + continue + + # Building all diffs between the current and expected policy + # XXX We simplify the comparison by assuming we have at most 1 + # sub_profile + if policy.get('subprofiles'): + if len(policy['subprofiles']) > 1: + raise ArgumentValueError('Multiple sub_profiles ({0}) are not ' + 'supported in the input policy') + subprofile = policy['subprofiles'][0] + current_subprofile = current_policy['subprofiles'][0] + capabilities_differ = list_diff(current_subprofile['capabilities'], + subprofile.get('capabilities', []), + key='id') + del policy['subprofiles'] + if subprofile.get('capabilities'): + del subprofile['capabilities'] + del current_subprofile['capabilities'] + # Get the subprofile diffs without the capability keys + subprofile_differ = recursive_diff(current_subprofile, + dict(subprofile)) + + del current_policy['subprofiles'] + policy_differ = recursive_diff(current_policy, policy) + if policy_differ.diffs or capabilities_differ.diffs or \ + subprofile_differ.diffs: + + changes_required = True + if __opts__['test']: + str_changes = [] + if policy_differ.diffs: + str_changes.extend( + [change for change in + policy_differ.changes_str.split('\n')]) + if subprofile_differ.diffs or \ + capabilities_differ.diffs: + + str_changes.append('subprofiles:') + if subprofile_differ.diffs: + str_changes.extend( + [' {0}'.format(change) for change in + subprofile_differ.changes_str.split('\n')]) + if capabilities_differ.diffs: + str_changes.append(' capabilities:') + str_changes.extend( + [' {0}'.format(change) for change in + capabilities_differ.changes_str2.split('\n')]) + comments.append( + 'State {0} will update the storage policy \'{1}\'' + ' on vCenter \'{2}\':\n{3}' + ''.format(name, policy['name'], vcenter, + '\n'.join( str_changes))) + else: + __salt__['vsphere.update_storage_policy']( + policy=current_policy['name'], + policy_dict=policy_copy, + service_instance=si) + comments.append('Updated the storage policy \'{0}\'' + 'in vCenter \'{1}\'' + ''.format(policy['name'], vcenter)) + log.info(comments[-1]) + + # Build new/old values to report what was changed + new_values = policy_differ.new_values + new_values['subprofiles'] = [subprofile_differ.new_values] + new_values['subprofiles'][0]['capabilities'] = \ + capabilities_differ.new_values + if not new_values['subprofiles'][0]['capabilities']: + del new_values['subprofiles'][0]['capabilities'] + if not new_values['subprofiles'][0]: + del new_values['subprofiles'] + old_values = policy_differ.old_values + old_values['subprofiles'] = [subprofile_differ.old_values] + old_values['subprofiles'][0]['capabilities'] = \ + capabilities_differ.old_values + if not old_values['subprofiles'][0]['capabilities']: + del old_values['subprofiles'][0]['capabilities'] + if not old_values['subprofiles'][0]: + del old_values['subprofiles'] + changes.append({'new': new_values, + 'old': old_values}) + else: + # No diffs found - no updates required + comments.append('Storage policy \'{0}\' is up to date. ' + 'Nothing to be done.'.format(policy['name'])) + __salt__['vsphere.disconnect'](si) + except CommandExecutionError as exc: + log.error('Error: {0}'.format(exc)) + if si: + __salt__['vsphere.disconnect'](si) + if not __opts__['test']: + ret['result'] = False + ret.update({'comment': exc.strerror, + 'result': False if not __opts__['test'] else None}) + return ret + if not changes_required: + # We have no changes + ret.update({'comment': ('All storage policy in vCenter ' + '\'{0}\' is correctly configured. ' + 'Nothing to be done.'.format(vcenter)), + 'result': True}) + else: + ret.update({'comment': '\n'.join(comments)}) + if __opts__['test']: + ret.update({'pchanges': {'storage_policies': changes}, + 'result': None}) + else: + ret.update({'changes': {'storage_policies': changes}, + 'result': True}) + return ret From 36fc89c9a2515c2ed4bdee6a375dae239377f403 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 20 Sep 2017 21:09:45 -0400 Subject: [PATCH 41/47] Added salt.states.pbm.default_storage_policy_assigned state that manages default storage policies to datastore assigments --- salt/states/pbm.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/salt/states/pbm.py b/salt/states/pbm.py index 54833151954b..e77f16f48bc5 100644 --- a/salt/states/pbm.py +++ b/salt/states/pbm.py @@ -437,3 +437,64 @@ def storage_policies_configured(name, policies): ret.update({'changes': {'storage_policies': changes}, 'result': True}) return ret + + +def default_storage_policy_assigned(name, policy, datastore): + ''' + Assigns a default storage policy to a datastore + + policy + Name of storage policy + + datastore + Name of datastore + ''' + log.info('Running state {0} for policy \'{1}\, datastore \'{2}\'.' + ''.format(name, policy, datastore)) + changes = {} + changes_required = False + ret = {'name': name, 'changes': {}, 'result': None, 'comment': None, + 'pchanges': {}} + si = None + try: + si = __salt__['vsphere.get_service_instance_via_proxy']() + existing_policy = \ + __salt__['vsphere.list_default_storage_policy_of_datastore']( + datastore=datastore, service_instance=si) + if existing_policy['name'] == policy: + comment = ('Storage policy \'{0}\' is already assigned to ' + 'datastore \'{1}\'. Nothing to be done.' + ''.format(policy, datastore)) + else: + changes_required = True + changes = { + 'default_storage_policy': {'old': existing_policy['name'], + 'new': policy}} + if (__opts__['test']): + comment = ('State {0} will assign storage policy \'{1}\' to ' + 'datastore \'{2}\'.').format(name, policy, + datastore) + else: + __salt__['vsphere.assign_default_storage_policy_to_datastore']( + policy=policy, datastore=datastore, service_instance=si) + comment = ('Storage policy \'{0} was assigned to datastore ' + '\'{1}\'.').format(policy, name) + log.info(comment) + except CommandExecutionError as exc: + log.error('Error: {}'.format(exc)) + if si: + __salt__['vsphere.disconnect'](si) + ret.update({'comment': exc.strerror, + 'result': False if not __opts__['test'] else None}) + return ret + ret['comment'] = comment + if changes_required: + if __opts__['test']: + ret.update({'result': None, + 'pchanges': changes}) + else: + ret.update({'result': True, + 'changes': changes}) + else: + ret['result'] = True + return ret From b6577e432894ad31cd3781201415b51e0af1c541 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Thu, 21 Sep 2017 12:54:23 -0400 Subject: [PATCH 42/47] pylint --- salt/modules/vsphere.py | 26 +++++++++++++------------- salt/proxy/vcenter.py | 8 ++++---- salt/states/pbm.py | 21 +++++---------------- salt/utils/pbm.py | 5 ++++- salt/utils/vmware.py | 2 +- tests/unit/modules/test_vsphere.py | 1 - tests/unit/utils/test_pbm.py | 14 +++++++------- 7 files changed, 34 insertions(+), 43 deletions(-) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index d4421ce1de21..bc59077b1af1 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4700,8 +4700,8 @@ def list_default_vsan_policy(service_instance=None): def_policies = [p for p in policies if p.systemCreatedProfileType == 'VsanDefaultProfile'] if not def_policies: - raise excs.VMwareObjectRetrievalError('Default VSAN policy was not ' - 'retrieved') + raise VMwareObjectRetrievalError('Default VSAN policy was not ' + 'retrieved') return _get_policy_dict(def_policies[0]) @@ -4854,8 +4854,8 @@ def update_storage_policy(policy, policy_dict, service_instance=None): profile_manager = salt.utils.pbm.get_profile_manager(service_instance) policies = salt.utils.pbm.get_storage_policies(profile_manager, [policy]) if not policies: - raise excs.VMwareObjectRetrievalError('Policy \'{0}\' was not found' - ''.format(policy)) + raise VMwareObjectRetrievalError('Policy \'{0}\' was not found' + ''.format(policy)) policy_ref = policies[0] policy_update_spec = pbm.profile.CapabilityBasedProfileUpdateSpec() log.trace('Setting policy values in policy_update_spec') @@ -4893,8 +4893,8 @@ def list_default_storage_policy_of_datastore(datastore, service_instance=None): ds_refs = salt.utils.vmware.get_datastores(service_instance, target_ref, datastore_names=[datastore]) if not ds_refs: - raise excs.VMwareObjectRetrievalError('Datastore \'{0}\' was not ' - 'found'.format(datastore)) + raise VMwareObjectRetrievalError('Datastore \'{0}\' was not ' + 'found'.format(datastore)) profile_manager = salt.utils.pbm.get_profile_manager(service_instance) policy = salt.utils.pbm.get_default_storage_policy_of_datastore( profile_manager, ds_refs[0]) @@ -4927,12 +4927,12 @@ def assign_default_storage_policy_to_datastore(policy, datastore, ''' log.trace('Assigning policy {0} to datastore {1}' ''.format(policy, datastore)) - profile_manager = utils_pbm.get_profile_manager(service_instance) + profile_manager = salt.utils.pbm.get_profile_manager(service_instance) # Find policy - policies = utils_pbm.get_storage_policies(profile_manager, [policy]) + policies = salt.utils.pbm.get_storage_policies(profile_manager, [policy]) if not policies: - raise excs.VMwareObjectRetrievalError('Policy \'{0}\' was not found' - ''.format(policy)) + raise VMwareObjectRetrievalError('Policy \'{0}\' was not found' + ''.format(policy)) policy_ref = policies[0] # Find datastore target_ref = _get_proxy_target(service_instance) @@ -4942,9 +4942,9 @@ def assign_default_storage_policy_to_datastore(policy, datastore, raise excs.VMwareObjectRetrievalError('Datastore \'{0}\' was not ' 'found'.format(datastore)) ds_ref = ds_refs[0] - utils_pbm.assign_default_storage_policy_to_datastore(profile_manager, - policy_ref, ds_ref) - return {'assign_storage_policy_to_datastore': True} + salt.utils.pbm.assign_default_storage_policy_to_datastore( + profile_manager, policy_ref, ds_ref) + return True @depends(HAS_PYVMOMI) diff --git a/salt/proxy/vcenter.py b/salt/proxy/vcenter.py index 7b9c9f95e304..5c5ad797d191 100644 --- a/salt/proxy/vcenter.py +++ b/salt/proxy/vcenter.py @@ -189,7 +189,7 @@ # Import Salt Libs import salt.exceptions -from salt.config.schemas.vcenter import VCenterProxySchema +from salt.config.schemas.vcenter import VCenterProxySchema from salt.utils.dictupdate import merge # This must be present or the Salt loader won't load this module. @@ -250,18 +250,18 @@ def init(opts): raise salt.exceptions.InvalidConfigError( 'Mechanism is set to \'userpass\' , but no ' '\'username\' key found in proxy config') - if not 'passwords' in proxy_conf: + if 'passwords' not in proxy_conf: raise salt.exceptions.InvalidConfigError( 'Mechanism is set to \'userpass\' , but no ' '\'passwords\' key found in proxy config') for key in ('username', 'passwords'): DETAILS[key] = proxy_conf[key] else: - if not 'domain' in proxy_conf: + if 'domain' not in proxy_conf: raise salt.exceptions.InvalidConfigError( 'Mechanism is set to \'sspi\' , but no ' '\'domain\' key found in proxy config') - if not 'principal' in proxy_conf: + if 'principal' not in proxy_conf: raise salt.exceptions.InvalidConfigError( 'Mechanism is set to \'sspi\' , but no ' '\'principal\' key found in proxy config') diff --git a/salt/states/pbm.py b/salt/states/pbm.py index e77f16f48bc5..bf54f620ad0f 100644 --- a/salt/states/pbm.py +++ b/salt/states/pbm.py @@ -95,32 +95,21 @@ # Import Python Libs from __future__ import absolute_import -import sys import logging -import json -import time import copy # Import Salt Libs from salt.exceptions import CommandExecutionError, ArgumentValueError -import salt.modules.vsphere as vsphere -from salt.utils import is_proxy from salt.utils.dictdiffer import recursive_diff from salt.utils.listdiffer import list_diff -# External libraries -try: - import jsonschema - HAS_JSONSCHEMA = True -except ImportError: - HAS_JSONSCHEMA = False - # Get Logging Started log = logging.getLogger(__name__) # TODO change with vcenter ALLOWED_PROXY_TYPES = ['esxcluster', 'vcenter'] LOGIN_DETAILS = {} + def __virtual__(): if HAS_JSONSCHEMA: return True @@ -297,7 +286,7 @@ def storage_policies_configured(name, policies): # All allowed proxies have a vcenter detail vcenter = __salt__['{0}.get_details'.format(proxy_type)]()['vcenter'] log.info('Running state \'{0}\' on vCenter ' - '\'{0}\''.format(name, vcenter)) + '\'{1}\''.format(name, vcenter)) si = __salt__['vsphere.get_service_instance_via_proxy']() current_policies = __salt__['vsphere.list_storage_policies']( policy_names=[policy['name'] for policy in policies], @@ -378,7 +367,7 @@ def storage_policies_configured(name, policies): 'State {0} will update the storage policy \'{1}\'' ' on vCenter \'{2}\':\n{3}' ''.format(name, policy['name'], vcenter, - '\n'.join( str_changes))) + '\n'.join(str_changes))) else: __salt__['vsphere.update_storage_policy']( policy=current_policy['name'], @@ -449,7 +438,7 @@ def default_storage_policy_assigned(name, policy, datastore): datastore Name of datastore ''' - log.info('Running state {0} for policy \'{1}\, datastore \'{2}\'.' + log.info('Running state {0} for policy \'{1}\', datastore \'{2}\'.' ''.format(name, policy, datastore)) changes = {} changes_required = False @@ -470,7 +459,7 @@ def default_storage_policy_assigned(name, policy, datastore): changes = { 'default_storage_policy': {'old': existing_policy['name'], 'new': policy}} - if (__opts__['test']): + if __opts__['test']: comment = ('State {0} will assign storage policy \'{1}\' to ' 'datastore \'{2}\'.').format(name, policy, datastore) diff --git a/salt/utils/pbm.py b/salt/utils/pbm.py index 17b25acecaa6..c7fa43eaa4bb 100644 --- a/salt/utils/pbm.py +++ b/salt/utils/pbm.py @@ -171,7 +171,7 @@ def get_policies_by_id(profile_manager, policy_ids): raise VMwareRuntimeError(exc.msg) -def get_storage_policies(profile_manager, policy_names=[], +def get_storage_policies(profile_manager, policy_names=None, get_all_policies=False): ''' Returns a list of the storage policies, filtered by name. @@ -181,6 +181,7 @@ def get_storage_policies(profile_manager, policy_names=[], policy_names List of policy names to filter by. + Default is None. get_all_policies Flag specifying to return all policies, regardless of the specified @@ -207,6 +208,8 @@ def get_storage_policies(profile_manager, policy_names=[], pbm.profile.ResourceTypeEnum.STORAGE] if get_all_policies: return policies + if not policy_names: + policy_names = [] return [p for p in policies if p.name in policy_names] diff --git a/salt/utils/vmware.py b/salt/utils/vmware.py index cbfb741dc0fc..018bb1041750 100644 --- a/salt/utils/vmware.py +++ b/salt/utils/vmware.py @@ -434,7 +434,7 @@ def get_new_service_instance_stub(service_instance, path, ns=None, #connection handshaking rule. We may need turn of the hostname checking #and client side cert verification context = None - if sys.version_info[:3] > (2,7,8): + if sys.version_info[:3] > (2, 7, 8): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE diff --git a/tests/unit/modules/test_vsphere.py b/tests/unit/modules/test_vsphere.py index 9ebad773631c..ed043f272832 100644 --- a/tests/unit/modules/test_vsphere.py +++ b/tests/unit/modules/test_vsphere.py @@ -648,7 +648,6 @@ def setUp(self): 'principal': 'fake_principal', 'domain': 'fake_domain'} - def tearDown(self): for attrname in ('esxi_host_details', 'esxi_vcenter_details', 'esxdatacenter_details', 'esxcluster_details'): diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index 4e08229e261a..aec9a51da5cd 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -10,7 +10,6 @@ import logging # Import Salt testing libraries -from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock, \ PropertyMock @@ -18,6 +17,7 @@ # Import Salt libraries from salt.exceptions import VMwareApiError, VMwareRuntimeError, \ VMwareObjectRetrievalError +from salt.ext.six.moves import range import salt.utils.pbm try: @@ -187,9 +187,9 @@ class GetCapabilityDefinitionsTestCase(TestCase): '''Tests for salt.utils.pbm.get_capability_definitions''' def setUp(self): self.mock_res_type = MagicMock() - self.mock_cap_cats =[MagicMock(capabilityMetadata=['fake_cap_meta1', - 'fake_cap_meta2']), - MagicMock(capabilityMetadata=['fake_cap_meta3'])] + self.mock_cap_cats = [MagicMock(capabilityMetadata=['fake_cap_meta1', + 'fake_cap_meta2']), + MagicMock(capabilityMetadata=['fake_cap_meta3'])] self.mock_prof_mgr = MagicMock( FetchCapabilityMetadata=MagicMock(return_value=self.mock_cap_cats)) patches = ( @@ -312,7 +312,7 @@ def setUp(self): self.mock_prof_mgr = MagicMock( QueryProfile=MagicMock(return_value=self.mock_policy_ids)) # Policies - self.mock_policies=[] + self.mock_policies = [] for i in range(4): mock_obj = MagicMock(resourceType=MagicMock( resourceType=pbm.profile.ResourceTypeEnum.STORAGE)) @@ -576,7 +576,7 @@ def test_get_policies_by_id(self): def test_no_policy_refs(self): mock_get_policies_by_id = MagicMock() - with path('salt.utils.pbm.get_policies_by_id', + with patch('salt.utils.pbm.get_policies_by_id', MagicMock(return_value=None)): with self.assertRaises(VMwareObjectRetrievalError) as excinfo: salt.utils.pbm.get_default_storage_policy_of_datastore( @@ -585,7 +585,7 @@ def test_no_policy_refs(self): 'Storage policy with id \'fake_policy_id\' was not ' 'found') - def test_no_policy_refs(self): + def test_return_policy_ref(self): mock_get_policies_by_id = MagicMock() ret = salt.utils.pbm.get_default_storage_policy_of_datastore( self.mock_prof_mgr, self.mock_ds) From f484bd52fd863af1ab55d729e9bf4182858cb6a6 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Thu, 21 Sep 2017 15:33:00 -0400 Subject: [PATCH 43/47] more pylint --- salt/states/pbm.py | 4 +--- tests/unit/utils/vmware/test_connection.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/salt/states/pbm.py b/salt/states/pbm.py index bf54f620ad0f..775b716f4461 100644 --- a/salt/states/pbm.py +++ b/salt/states/pbm.py @@ -111,9 +111,7 @@ def __virtual__(): - if HAS_JSONSCHEMA: - return True - return False + return True def mod_init(low): diff --git a/tests/unit/utils/vmware/test_connection.py b/tests/unit/utils/vmware/test_connection.py index dd357d487087..d8afbb0504c1 100644 --- a/tests/unit/utils/vmware/test_connection.py +++ b/tests/unit/utils/vmware/test_connection.py @@ -25,7 +25,7 @@ from salt.ext import six try: - from pyVmomi import vim, vmodl, VmomiSupport + from pyVmomi import vim, vmodl HAS_PYVMOMI = True except ImportError: HAS_PYVMOMI = False From e1bfe248915d6fc623e3bfb4a96c008821dd760a Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Fri, 22 Sep 2017 08:59:33 -0400 Subject: [PATCH 44/47] Removed excs reference from new methods in salt.modules.vsphere --- salt/modules/vsphere.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index bc59077b1af1..0c923858042d 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -4939,8 +4939,8 @@ def assign_default_storage_policy_to_datastore(policy, datastore, ds_refs = salt.utils.vmware.get_datastores(service_instance, target_ref, datastore_names=[datastore]) if not ds_refs: - raise excs.VMwareObjectRetrievalError('Datastore \'{0}\' was not ' - 'found'.format(datastore)) + raise VMwareObjectRetrievalError('Datastore \'{0}\' was not ' + 'found'.format(datastore)) ds_ref = ds_refs[0] salt.utils.pbm.assign_default_storage_policy_to_datastore( profile_manager, policy_ref, ds_ref) From 4ff745d2c5d30805222d9d7981aa7302b94c4542 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Fri, 22 Sep 2017 15:15:47 -0400 Subject: [PATCH 45/47] Added python/pyvmomi compatibility check to salt.states.pbm + removed reference to Python 2.6 --- salt/states/pbm.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/salt/states/pbm.py b/salt/states/pbm.py index 775b716f4461..00945fc65cf6 100644 --- a/salt/states/pbm.py +++ b/salt/states/pbm.py @@ -97,20 +97,34 @@ from __future__ import absolute_import import logging import copy +import sys # Import Salt Libs from salt.exceptions import CommandExecutionError, ArgumentValueError from salt.utils.dictdiffer import recursive_diff from salt.utils.listdiffer import list_diff +# External libraries +try: + from pyVmomi import VmomiSupport + HAS_PYVMOMI = True +except ImportError: + HAS_PYVMOMI = False + # Get Logging Started log = logging.getLogger(__name__) -# TODO change with vcenter -ALLOWED_PROXY_TYPES = ['esxcluster', 'vcenter'] -LOGIN_DETAILS = {} def __virtual__(): + if not HAS_PYVMOMI: + return False, 'State module did not load: pyVmomi not found' + + # We check the supported vim versions to infer the pyVmomi version + if 'vim25/6.0' in VmomiSupport.versionMap and \ + sys.version_info > (2, 7) and sys.version_info < (2, 7, 9): + + return False, ('State module did not load: Incompatible versions ' + 'of Python and pyVmomi present. See Issue #29537.') return True From ac79f89ffa92bbb2c27c3a1418f6fb63b93838e0 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Tue, 26 Sep 2017 04:59:00 -0400 Subject: [PATCH 46/47] Fixed utils.pbm unit tests --- tests/unit/utils/test_pbm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils/test_pbm.py b/tests/unit/utils/test_pbm.py index aec9a51da5cd..6c2be0f9b581 100644 --- a/tests/unit/utils/test_pbm.py +++ b/tests/unit/utils/test_pbm.py @@ -214,7 +214,7 @@ def test_get_res_type(self): def test_fetch_capabilities(self): salt.utils.pbm.get_capability_definitions(self.mock_prof_mgr) - self.mock_prof_mgr.FetchCapabilityMetadata.assert_callend_once_with( + self.mock_prof_mgr.FetchCapabilityMetadata.assert_called_once_with( self.mock_res_type) def test_fetch_capabilities_raises_no_permissions(self): @@ -268,7 +268,7 @@ def tearDown(self): def test_retrieve_policies(self): salt.utils.pbm.get_policies_by_id(self.mock_prof_mgr, self.policy_ids) - self.mock_prof_mgr.RetrieveContent.assert_callend_once_with( + self.mock_prof_mgr.RetrieveContent.assert_called_once_with( self.policy_ids) def test_retrieve_policies_raises_no_permissions(self): From 5c3109ff071a7e1b18680a9217b0539d2c9ae4e1 Mon Sep 17 00:00:00 2001 From: Alexandru Bleotu Date: Wed, 27 Sep 2017 12:55:46 -0400 Subject: [PATCH 47/47] Removed commented imports --- salt/utils/vmware.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/utils/vmware.py b/salt/utils/vmware.py index 018bb1041750..b0552996e32d 100644 --- a/salt/utils/vmware.py +++ b/salt/utils/vmware.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- ''' -import sys -import ssl Connection library for VMware .. versionadded:: 2015.8.2