diff --git a/Makefile b/Makefile index 6cdc17131..8d4b2a1da 100644 --- a/Makefile +++ b/Makefile @@ -17,13 +17,12 @@ endif pex: pip install --upgrade pip - pip install --upgrade wheel requests pex + pip install --upgrade 'wheel<0.30.0' requests pex -$(RM) $(output_dir) pex -v -o $(output_dir)/$(output_filename)$(output_file_extension) -m user_sync.app \ -f $(prebuilt_dir) \ --disable-cache \ --not-zip-safe . - -$(RM) wheelhouse test: nosetests --no-byte-compile tests diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5078ae263..a6bb52b7c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,34 +1,31 @@ -# Release Notes for User Sync Tool Version 2.2.1 +# Release Notes for User Sync Tool Version 2.2.2 -These notes apply to v2.2.1 of 2017-08-30. +These notes apply to v2.2.2rc2 of 2017-10-29. ## New Features -[#266](https://github.com/adobe-apiplatform/user-sync.py/issues/266): Extended attribute values (defined in extensions) can now be multi-valued. The type of the attribute value in the `source_attributes` dictionary will be: -* `None` if the attribute has no value; -* a `str` (or `unicode` in py2) if the attribute has one value; -* a `list` of `str` (or `unicode` in py2) if the attribute has multiple values. - -[#268](https://github.com/adobe-apiplatform/user-sync.py/issues/268): To make sure users get all the right overlapping entitlements associated with mapped user groups, `--strategy push` now does group removals before group adds. +[#294](https://github.com/adobe-apiplatform/user-sync.py/issues/294): Show statistics about users added to secondaries. ## Bug Fixes -[#257](https://github.com/adobe-apiplatform/user-sync.py/issues/257): Catch exceptions thrown by umapi-client when creating actions. - -[#258](https://github.com/adobe-apiplatform/user-sync.py/issues/258): Correctly decrypt private keys in py3. +[#283](https://github.com/adobe-apiplatform/user-sync.py/issues/283): Don't import keyring unless needed. -[#260](https://github.com/adobe-apiplatform/user-sync.py/issues/260): Make sure the requests library is loaded when using pex on Windows. +[#286](https://github.com/adobe-apiplatform/user-sync.py/issues/286): Allow specifying attributes for Adobe IDs. -[#265](https://github.com/adobe-apiplatform/user-sync.py/issues/265): Extended attributes in extensions couldn't be fetched unless they had non-ascii names. +[#288](https://github.com/adobe-apiplatform/user-sync.py/issues/288): Escape special characters in user input to LDAP queries. -[#269](https://github.com/adobe-apiplatform/user-sync.py/issues/269): When using `--strategy sync`, new users created in secondary organizations were not being added to any groups. +[#293](https://github.com/adobe-apiplatform/user-sync.py/issues/293): Don't crash when existing users are added to secondaries. ## Compatibility with Prior Versions -There are no functional changes from prior versions. +There are no interface changes from prior versions. ## Known Issues -Because the release on Windows is built with a pre-compiled version of pyldap, we have to specify a specific version to be used in each release. This not always be the latest version. +The nosetests are broken in this release candidate. + +Because the release on Windows is built with a pre-compiled version of pyldap, we have to specify a specific version to be used in each release. This may not always be the latest version. On the Win64 platform, there are very long pathnames embedded in the released build artifact `user-sync.pex`, which will cause problems unless you are on Windows 10 and are either running Python 3.6 or have enabled long pathnames system-wide (as described in this [Microsoft Dev Center article](https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)). To work around this issue on older platforms, set the `PEX_ROOT` environment variable (as described [in the docs here](https://adobe-apiplatform.github.io/user-sync.py/en/user-manual/setup_and_installation.html)) to be a very short path (e.g., `set PEX_ROOT=C:\pex`). + +Each release on each platform is built with a specific version of Python. Typically this is the latest available (from the OS vendor, if they provide one) for that platform. In general, and especially on Windows, you should use the same Python to run User Sync as it was built with. diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html index a4b5377f3..7d670fcda 100644 --- a/docs/_layouts/default.html +++ b/docs/_layouts/default.html @@ -28,8 +28,11 @@ - {% assign current_level = my_page.nav_level %} + {% assign ub = current_level | minus: 1 %} + {% for i in (my_page.nav_level .. ub) %} + + {% endfor %} + {% assign current_level = my_page.nav_level %} {% endif %} {% if my_page.url == page.url %}
  • {{ my_page.nav_link | escape }}
    • diff --git a/examples/config files - basic/3 connector-ldap.yml b/examples/config files - basic/3 connector-ldap.yml index c84a85299..f425653dc 100755 --- a/examples/config files - basic/3 connector-ldap.yml +++ b/examples/config files - basic/3 connector-ldap.yml @@ -52,7 +52,10 @@ search_page_size: 200 require_tls_cert: False # (optional) all_users_filter (default value given below) -# all_users_filter specifies the query used to find all users in the directory. +# Use this filter to control exactly which LDAP resources are considered for synchronization. +# If a user is not accepted by this filter he will never be seen by the User Sync tool, even if +# your command line specifies --users all or he is in a group you have specified in your +# command line (--users group g1) or in your configuration file (for group mapping). # The default value specified here is appropriate for Active Directory, which has a # special field that is used to enable and disable users. The value for OpenLDAP # directories might be much simpler: "(&(objectClass=person)(objectClass=top))" diff --git a/setup.py b/setup.py index 98be9ce81..4e287b022 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ 'pycryptodome', 'pyldap==2.4.37', 'PyYAML', - 'umapi-client>=2.7', + 'umapi-client>=2.8', 'psutil', 'keyring', 'six' diff --git a/user_sync/config.py b/user_sync/config.py index bcbd24aae..be2f01824 100644 --- a/user_sync/config.py +++ b/user_sync/config.py @@ -25,7 +25,6 @@ import subprocess import types -import keyring import yaml import six @@ -600,6 +599,7 @@ def get_credential(self, name, user_name, none_allowed=False): raise AssertionException('%s: cannot contain setting for both "%s" and "%s"' % (scope, name, keyring_name)) if secure_value_key: try: + import keyring value = keyring.get_password(service_name=secure_value_key, username=user_name) except Exception as e: raise AssertionException('%s: Error accessing secure storage: %s' % (scope, e)) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index 9479094b0..c4628aa47 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -133,7 +133,7 @@ def load_users_and_groups(self, groups, extended_attributes, all_users): if not group_dn: self.logger.warning("No group found for: %s", group) continue - group_member_subfilter = group_member_filter_format.format(group_dn=group_dn) + group_member_subfilter = self.format_ldap_query_string(group_member_filter_format, group_dn=group_dn) if not group_member_subfilter.startswith('('): group_member_subfilter = six.text_type('(') + group_member_subfilter + six.text_type(')') user_subfilter = all_users_filter @@ -178,8 +178,9 @@ def find_ldap_group_dn(self, group): base_dn = six.text_type(options['base_dn']) group_filter_format = six.text_type(options['group_filter_format']) try: + filter_string = self.format_ldap_query_string(group_filter_format, group=group) res = connection.search_s(base_dn, ldap.SCOPE_SUBTREE, - filterstr=group_filter_format.format(group=group), attrsonly=1) + filterstr=filter_string, attrsonly=1) except Exception as e: raise AssertionException('Unexpected LDAP failure reading group info: %s' % e) group_dn = None @@ -324,6 +325,34 @@ def iter_search_result(self, base_dn, scope, filter_string, attributes): connection.abandon(msgid) raise + @staticmethod + def format_ldap_query_string(query, **kwargs): + """ + Escape LDAP special characters that may appear in injected query strings + Should be used with any string that will be injected into an LDAP query. + :param query: + :param kwargs: + :return: + """ + # See http://www.rfc-editor.org/rfc/rfc4515.txt + escape_chars = six.text_type('*()\\&|<>~!:') + escaped_args = {} + # kwargs is a dict that would normally be passed to string.format + for k, v in six.iteritems(kwargs): + # LDAP special characters are escaped in the general format '\' + hex(char) + # we need to run through the string char by char and if the char exists in + # the escape_char list, get the ord of it (decimal ascii value), convert it to hex, and + # replace the '0x' with '\' + escaped_list = [] + for c in v: + if c in escape_chars: + replace = six.text_type(hex(ord(c))).replace('0x', '\\') + escaped_list.append(replace) + else: + escaped_list.append(c) + escaped_args[k] = six.text_type('').join(escaped_list) + return query.format(**escaped_args) + class LDAPValueFormatter(object): encoding = 'utf8' diff --git a/user_sync/connector/umapi.py b/user_sync/connector/umapi.py index 2ed87f61d..3a7cc01eb 100644 --- a/user_sync/connector/umapi.py +++ b/user_sync/connector/umapi.py @@ -178,9 +178,6 @@ def update_user(self, attributes): """ :type attributes: dict """ - if self.identity_type == user_sync.identity_type.ADOBEID_IDENTITY_TYPE: - # shouldn't happen, but ignore it if it does - return if attributes is not None and len(attributes) > 0: params = self.convert_user_attributes_to_params(attributes) self.do_list.append(('update', params)) @@ -212,14 +209,7 @@ def add_user(self, attributes): """ :type attributes: dict """ - if self.identity_type == user_sync.identity_type.ADOBEID_IDENTITY_TYPE: - email = self.email if self.email else self.username - if not email: - error_message = "ERROR: you must specify an email with an Adobe ID" - raise AssertionException(error_message) - params = self.convert_user_attributes_to_params({'email': email}) - else: - params = self.convert_user_attributes_to_params(attributes) + params = self.convert_user_attributes_to_params(attributes) on_conflict_value = None option = params.pop('option', None) diff --git a/user_sync/rules.py b/user_sync/rules.py index ad759d6f6..4db45ffd6 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -68,14 +68,15 @@ def __init__(self, caller_options): # counters for action summary log self.action_summary = { # these are in alphabetical order! Always add new ones that way! - 'adobe_strays_processed': 0, - 'adobe_users_created': 0, - 'adobe_users_excluded': 0, - 'adobe_users_read': 0, - 'adobe_users_unchanged': 0, - 'adobe_users_updated': 0, 'directory_users_read': 0, 'directory_users_selected': 0, + 'excluded_user_count': 0, + 'primary_strays_processed': 0, + 'primary_users_created': 0, + 'primary_users_read': 0, + 'secondary_users_created': 0, + 'unchanged_user_count': 0, + 'updated_user_count': 0, } self.logger = logger = logging.getLogger('processor') @@ -92,12 +93,16 @@ def __init__(self, caller_options): # of primary-umapi users, who are presumed to be in primary-umapi domains. # So instead of keeping track of excluded users in the primary umapi, # we keep track of included users, so we can match them against users - # in the secondary umapis (and exclude all that don't match). Finally, - # we keep track of user keys (in any umapi) that we have updated, so + # in the secondary umapis (and exclude all that don't match). We track + # primary users created and secondary users created so that we can figure + # out which existing users were created in the secondaries only. Finally, + # we keep track of user keys that we have updated in any umapi, so that # we can correctly report their count. - self.adobe_user_count = 0 + self.primary_user_count = 0 self.included_user_keys = set() self.excluded_user_count = 0 + self.primary_users_created = set() + self.secondary_users_created = set() self.updated_user_keys = set() # stray key input path comes in, stray_list_output_path goes out @@ -109,13 +114,14 @@ def __init__(self, caller_options): # determine what processing is needed on strays self.will_manage_strays = (options['manage_groups'] or options['disentitle_strays'] or options['remove_strays'] or options['delete_strays']) - self.will_process_strays = (not options['exclude_strays']) and (options['stray_list_output_path'] or - self.will_manage_strays) + self.exclude_strays = options['exclude_strays'] + self.will_process_strays = ((not self.exclude_strays) and + (options['stray_list_output_path'] or self.will_manage_strays)) # specifying a push strategy disables a lot of processing - self.sync_umapi = True + self.push_umapi = False if options['strategy'] == 'push': - self.sync_umapi = False + self.push_umapi = True self.will_manage_strays = False self.will_process_strays = False @@ -163,13 +169,10 @@ def run(self, directory_groups, directory_connector, umapi_connectors): self.read_desired_user_groups(directory_groups, directory_connector) load_directory_stats.log_end(logger) - umapi_stats = JobStats('Sync with UMAPI' if self.sync_umapi else 'Push to UMAPI', divider="-") + umapi_stats = JobStats('Push to UMAPI' if self.push_umapi else 'Sync with UMAPI', divider="-") umapi_stats.log_start(logger) if directory_connector is not None: - if self.sync_umapi: - self.sync_umapi_users(umapi_connectors) - else: - self.push_umapi_users(umapi_connectors) + self.sync_umapi_users(umapi_connectors) if self.will_process_strays: self.process_strays(umapi_connectors) umapi_connectors.execute_actions() @@ -188,44 +191,50 @@ def log_action_summary(self, umapi_connectors): self.action_summary['directory_users_read'] = len(self.directory_user_by_user_key) self.action_summary['directory_users_selected'] = len(self.filtered_directory_user_by_user_key) # find the total number of adobe users and excluded users - self.action_summary['adobe_users_read'] = self.adobe_user_count - self.action_summary['adobe_users_excluded'] = self.excluded_user_count - self.action_summary['adobe_users_updated'] = len(self.updated_user_keys) + self.action_summary['primary_users_read'] = self.primary_user_count + self.action_summary['excluded_user_count'] = self.excluded_user_count + self.action_summary['updated_user_count'] = len(self.updated_user_keys) # find out the number of users that have no changes; this depends on whether - # we actually read the directory or read an input file. So there are two cases: - if self.action_summary['adobe_users_read'] == 0: - self.action_summary['adobe_users_unchanged'] = 0 + # we actually read the directory or read a key file. So there are two cases: + if self.action_summary['primary_users_read'] == 0: + self.action_summary['unchanged_user_count'] = 0 else: - self.action_summary['adobe_users_unchanged'] = ( - self.action_summary['adobe_users_read'] - - self.action_summary['adobe_users_excluded'] - - self.action_summary['adobe_users_updated'] - - self.action_summary['adobe_strays_processed'] + self.action_summary['unchanged_user_count'] = ( + self.action_summary['primary_users_read'] - + self.action_summary['excluded_user_count'] - + self.action_summary['updated_user_count'] - + self.action_summary['primary_strays_processed'] ) - if self.options['test_mode']: - header = '- Action Summary (TEST MODE) -' - else: - header = '------- Action Summary -------' - logger.info('---------------------------' + header + '---------------------------') + # find out the number of users created in the primary and secondary umapis + self.action_summary['primary_users_created'] = len(self.primary_users_created) + self.action_summary['secondary_users_created'] = len(self.secondary_users_created) # English text description for action summary log. # The action summary will be shown the same order as they are defined in this list - if self.sync_umapi: + if self.push_umapi: action_summary_description = [ ['directory_users_read', 'Number of directory users read'], ['directory_users_selected', 'Number of directory users selected for input'], - ['adobe_users_read', 'Number of Adobe users read'], - ['adobe_users_excluded', 'Number of Adobe users excluded from updates'], - ['adobe_users_unchanged', 'Number of non-excluded Adobe users with no changes'], - ['adobe_users_created', 'Number of new Adobe users added'], - ['adobe_users_updated', 'Number of matching Adobe users updated'], + ['primary_users_created', 'Number of directory users pushed to Adobe'], ] + if umapi_connectors.get_secondary_connectors(): + action_summary_description += [ + ['secondary_users_created', 'Number of Adobe users pushed to secondaries'], + ] else: action_summary_description = [ ['directory_users_read', 'Number of directory users read'], ['directory_users_selected', 'Number of directory users selected for input'], - ['adobe_users_created', 'Number of directory users pushed to Adobe'], + ['primary_users_read', 'Number of Adobe users read'], + ['excluded_user_count', 'Number of Adobe users excluded from updates'], + ['unchanged_user_count', 'Number of non-excluded Adobe users with no changes'], + ['primary_users_created', 'Number of new Adobe users added'], + ['updated_user_count', 'Number of matching Adobe users updated'], ] + if umapi_connectors.get_secondary_connectors(): + action_summary_description += [ + ['secondary_users_created', 'Number of Adobe users added to secondaries'], + ] if self.will_process_strays: if self.options['delete_strays']: action = 'deleted' @@ -235,7 +244,7 @@ def log_action_summary(self, umapi_connectors): action = 'removed from all groups' else: action = 'with groups processed' - action_summary_description.append(['adobe_strays_processed', 'Number of Adobe-only users ' + action]) + action_summary_description.append(['primary_strays_processed', 'Number of Adobe-only users ' + action]) # prepare the network summary umapi_summary_format = 'Number of%s%s UMAPI actions sent (total, success, error)' @@ -257,7 +266,13 @@ def log_action_summary(self, umapi_connectors): umapi_summary_description = umapi_summary_format % (spacer, name) if len(umapi_summary_description) > pad: pad = len(umapi_summary_description) - # and then we use it + + # do the report + if self.options['test_mode']: + header = '- Action Summary (TEST MODE) -' + else: + header = '------- Action Summary -------' + logger.info('---------------------------' + header + '---------------------------') for action_description in action_summary_description: description = action_description[1].rjust(pad, ' ') action_count = self.action_summary[action_description[0]] @@ -268,6 +283,12 @@ def log_action_summary(self, umapi_connectors): logger.info(' %s: (%s, %s, %s)', description, sent, sent - errors, errors) logger.info('------------------------------------------------------------------------------------') + def is_primary_org(self, umapi_info): + return umapi_info.get_name() == PRIMARY_UMAPI_NAME + + def will_update_user_info(self, umapi_info): + return self.options['update_user_info'] and self.is_primary_org(umapi_info) + def will_manage_groups(self): return self.options['manage_groups'] @@ -327,7 +348,7 @@ def read_desired_user_groups(self, mappings, directory_connector): self.after_mapping_hook_scope['source_groups'] = set() self.after_mapping_hook_scope['target_groups'] = set() for group in directory_user['groups']: - self.after_mapping_hook_scope['source_groups'].add(group) # this is a directory group name + self.after_mapping_hook_scope['source_groups'].add(group) # this is a directory group name adobe_groups = mappings.get(group) if adobe_groups is not None: for adobe_group in adobe_groups: @@ -348,7 +369,7 @@ def read_desired_user_groups(self, mappings, directory_connector): # invoke the customer's hook code self.log_after_mapping_hook_scope(before_call=True) - exec (options['after_mapping_hook'], self.after_mapping_hook_scope) + exec(options['after_mapping_hook'], self.after_mapping_hook_scope) self.log_after_mapping_hook_scope(after_call=True) # copy modified attributes back to the user object @@ -367,7 +388,7 @@ def read_desired_user_groups(self, mappings, directory_connector): self.logger.debug('Group work list: %s', dict([(umapi_name, umapi_info.get_desired_groups_by_user_key()) for umapi_name, umapi_info in six.iteritems(self.umapi_info_by_name)])) - + def is_directory_user_in_groups(self, directory_user, groups): """ :type directory_user: dict @@ -384,56 +405,50 @@ def is_directory_user_in_groups(self, directory_user, groups): def sync_umapi_users(self, umapi_connectors): """ This is where we actually "do the sync"; that is, where we match users on the two sides. - When we get here, we have loaded all the directory users *and* we have loaded all the adobe users, - and (conceptually) we match them up, updating the adobe users that match, marking the umapi - users that don't match for deletion, and adding adobe users for the directory users that didn't match. - What makes the code here more complex is that, instead of looping over users just once and - updating each user in all of the umapi connectors at that time, we instead loop over users - once per umapi for which we have a umapi connector, and we do the matching logic for each of - those umapis. + When we get here, we have loaded all the directory users. Then, for each umapi connector, + we sync the directory users against the user in the umapi connector, yielding a set of + unmatched directory users which we then create on the Adobe side. :type umapi_connectors: UmapiConnectors """ + if self.push_umapi: + verb = "Push" + else: + verb = "Sync" + # first sync the primary connector, so the users get created in the primary if umapi_connectors.get_secondary_connectors(): - self.logger.debug('Syncing users to primary umapi...') + self.logger.debug('%sing users to primary umapi...', verb) else: - self.logger.debug('Syncing users to umapi...') - primary_umapi_info = self.get_umapi_info(PRIMARY_UMAPI_NAME) - - # Loop over users and compare then and process differences - primary_adds_by_user_key = self.update_umapi_users_for_connector(primary_umapi_info, - umapi_connectors.get_primary_connector()) - - # Handle creates for new users. This also drives adding the new user to the secondaries, - # but the secondary adobe groups will be managed below in the usual way. + self.logger.debug('%sing users to umapi...', verb) + umapi_info, umapi_connector = self.get_umapi_info(PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() + if self.push_umapi: + primary_adds_by_user_key = umapi_info.get_desired_groups_by_user_key() + else: + primary_adds_by_user_key = self.update_umapi_users_for_connector(umapi_info, umapi_connector) for user_key, groups_to_add in six.iteritems(primary_adds_by_user_key): - self.add_umapi_user(user_key, groups_to_add, umapi_connectors) - # we just did a bunch of adds, we need to flush the connections before we can sync groups - umapi_connectors.execute_actions() + # We always create every user in the primary umapi, because it's believed to own the directories. + self.logger.info('Creating user with user key: %s', user_key) + self.primary_users_created.add(user_key) + self.create_umapi_user(user_key, groups_to_add, umapi_info, umapi_connector) - # Now manage the adobe groups in the secondaries + # then sync the secondary connectors for umapi_name, umapi_connector in six.iteritems(umapi_connectors.get_secondary_connectors()): - secondary_umapi_info = self.get_umapi_info(umapi_name) - if len(secondary_umapi_info.get_mapped_groups()) == 0: + umapi_info = self.get_umapi_info(umapi_name) + if len(umapi_info.get_mapped_groups()) == 0: continue - self.logger.debug('Syncing users to secondary umapi %s...', umapi_name) - secondary_updates_by_user_key = self.update_umapi_users_for_connector(secondary_umapi_info, umapi_connector) - if secondary_updates_by_user_key: - self.logger.critical("Shouldn't happen! In secondary umapi %s, the following users were not found: %s", - umapi_name, secondary_updates_by_user_key.keys()) - - def push_umapi_users(self, umapi_connectors): - """ - This is where we push directory users to the Adobe side "as is". - :type umapi_connectors: UmapiConnectors - """ - if umapi_connectors.get_secondary_connectors(): - self.logger.debug('Pushing users to primary umapi...') - else: - self.logger.debug('Pushing users to umapi...') - primary_umapi_info = self.get_umapi_info(PRIMARY_UMAPI_NAME) - # Create all the users, putting them in their groups - for user_key, groups_to_add in six.iteritems(primary_umapi_info.get_desired_groups_by_user_key()): - self.add_umapi_user(user_key, groups_to_add, umapi_connectors) + self.logger.debug('%sing users to secondary umapi %s...', verb, umapi_name) + if self.push_umapi: + secondary_adds_by_user_key = umapi_info.get_desired_groups_by_user_key() + else: + secondary_adds_by_user_key = self.update_umapi_users_for_connector(umapi_info, umapi_connector) + for user_key, groups_to_add in six.iteritems(secondary_adds_by_user_key): + # We only create users who have group mappings in the secondary umapi + if groups_to_add: + self.logger.info('Adding user to umapi %s with user key: %s', umapi_name, user_key) + self.secondary_users_created.add(user_key) + if user_key not in self.primary_users_created: + # We pushed an existing user to a secondary in order to update his groups + self.updated_user_keys.add(user_key) + self.create_umapi_user(user_key, groups_to_add, umapi_info, umapi_connector) def is_selected_user_key(self, user_key): """ @@ -488,18 +503,20 @@ def process_strays(self, umapi_connectors): if stray_count > max_missing: self.logger.critical('Unable to process Adobe-only users, as their count (%s) is larger ' 'than the max_adobe_only_users setting (%d)', stray_count, max_missing) - self.action_summary['adobe_strays_processed'] = 0 + self.action_summary['primary_strays_processed'] = 0 return - self.action_summary['adobe_strays_processed'] = stray_count self.logger.debug("Processing Adobe-only users...") self.manage_strays(umapi_connectors) def manage_strays(self, umapi_connectors): """ Manage strays. This doesn't require having loaded users from the umapi. - Management of groups, removal of entitlements and removal from umapi are + Management of groups, removal of entitlements and removal from umapi are processed against every secondary umapi, whereas account deletion is only done against the primary umapi. + Because all directory users are assumed to be in the primary (as the owning org of the directory), + we don't pay any attention to stray users in the secondary who aren't in the primary. Instead, + we assume that they are users whose directory is owned by the secondary. :type umapi_connectors: UmapiConnectors """ # figure out what management to do @@ -510,6 +527,7 @@ def manage_strays(self, umapi_connectors): # all our processing is controlled by the strays in the primary organization primary_strays = self.get_stray_keys() + self.action_summary['primary_strays_processed'] = len(primary_strays) # convenience function to get umapi Commands given a user key def get_commands(key): @@ -592,82 +610,63 @@ def get_identity_type_from_umapi_user(self, umapi_user): self.logger.error('Found adobe user with no identity type, using %s: %s', identity_type, umapi_user) return identity_type - def create_commands_from_directory_user(self, directory_user, identity_type=None): + def create_umapi_commands_for_directory_user(self, directory_user, do_update=False): """ + Make the umapi commands to create this user, based on his directory attributes and type. + Update the attributes of an existing user if do_update is True. :type directory_user: dict - :type identity_type: str + :type do_update: bool """ - if identity_type is None: - identity_type = self.get_identity_type_from_directory_user(directory_user) + identity_type = self.get_identity_type_from_directory_user(directory_user) commands = user_sync.connector.umapi.Commands(identity_type, directory_user['email'], directory_user['username'], directory_user['domain']) - return commands - - def add_umapi_user(self, user_key, groups_to_add, umapi_connectors): - """ - Add the user to the primary umapi with the given groups, and create the user in any secondaries - in which he should be in a group. If we are syncing, that's all we do. If we are pushing rather - than syncing, we also add the user to the correct group in the secondaries, and we remove - the user from any mapped groups he shouldn't be in both in the primary and in the secondaries. - (This way, when we push blindly, we manage his entire set of mapped groups rather than just some.) - :type user_key: str - :type groups_to_add: list - :type umapi_connectors: UmapiConnectors - """ - # Check to see what we're updating - options = self.options - update_user_info = options['update_user_info'] - manage_groups = self.will_manage_groups() - doing_push = not self.sync_umapi - - # put together the user's attributes - directory_user = self.directory_user_by_user_key[user_key] - identity_type = self.get_identity_type_from_directory_user(directory_user) - primary_commands = self.create_commands_from_directory_user(directory_user, identity_type) attributes = self.get_user_attributes(directory_user) # check whether the country is set in the directory, use default if not country = directory_user['country'] if not country: - country = options['default_country_code'] + country = self.options['default_country_code'] if not country: if identity_type == user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE: # Enterprise users are allowed to have undefined country country = 'UD' else: - self.logger.error("Federated user cannot be added without a specified country code: %s", user_key) + self.logger.error("User cannot be added without a specified country code: %s", directory_user) return attributes['country'] = country if attributes.get('firstname') is None: attributes.pop('firstname', None) if attributes.get('lastname') is None: attributes.pop('lastname', None) - attributes['option'] = 'updateIfAlreadyExists' if update_user_info else 'ignoreIfAlreadyExists' - # add the user to primary with groups - self.logger.info('Adding directory user with user key: %s', user_key) - self.action_summary['adobe_users_created'] += 1 - primary_commands.add_user(attributes) - if manage_groups: - if doing_push: - groups_to_remove = self.get_umapi_info(PRIMARY_UMAPI_NAME).get_mapped_groups() - groups_to_add - primary_commands.remove_groups(groups_to_remove) - primary_commands.add_groups(groups_to_add) - umapi_connectors.get_primary_connector().send_commands(primary_commands) - # add the user to secondaries, maybe with groups - attributes['option'] = 'ignoreIfAlreadyExists' # can only update in the owning org - for umapi_name, umapi_connector in six.iteritems(umapi_connectors.secondary_connectors): - secondary_umapi_info = self.get_umapi_info(umapi_name) - # only add the user to this secondary if he is in groups in this secondary - groups_to_add = secondary_umapi_info.get_desired_groups(user_key) - if groups_to_add: - self.logger.info('Adding directory user to %s with user key: %s', umapi_name, user_key) - secondary_commands = self.create_commands_from_directory_user(directory_user, identity_type) - secondary_commands.add_user(attributes) - if manage_groups: - if doing_push: - groups_to_remove = secondary_umapi_info.get_mapped_groups() - groups_to_add - secondary_commands.remove_groups(groups_to_remove) - secondary_commands.add_groups(groups_to_add) - umapi_connector.send_commands(secondary_commands) + if do_update: + attributes['option'] = 'updateIfAlreadyExists' + else: + attributes['option'] = 'ignoreIfAlreadyExists' + commands.add_user(attributes) + return commands + + def create_umapi_user(self, user_key, groups_to_add, umapi_info, umapi_connector): + """ + Add the user to the org on the receiving end of the given umapi connector. + If the connector is the primary connector, we ask to update the user's attributes because + we believe the primary org owns the directory where users accounts are. Otherwise, + we send the user's attributes over, but we don't update them if the user exists. + If groups_to_add is specified, and we are managing groups, we give the user those groups. + If we are pushing, we also remove the user from any mapped groups not in groups_to_add. + (This way, when we push blindly, we manage the entire set of mapped groups.) + :type user_key: str + :type update_attributes: bool + :type groups_to_add: set + :type umapi_info: UmapiTargetInfo + :type umapi_connector: user_sync.connector.umapi.UmapiConnector + """ + directory_user = self.directory_user_by_user_key[user_key] + commands = self.create_umapi_commands_for_directory_user(directory_user, self.will_update_user_info(umapi_info)) + if self.will_manage_groups(): + if self.push_umapi: + groups_to_remove = umapi_info.get_mapped_groups() - groups_to_add + commands.remove_groups(groups_to_remove) + commands.add_groups(groups_to_add) + umapi_connector.send_commands(commands) def update_umapi_user(self, umapi_info, user_key, umapi_connector, attributes_to_update=None, groups_to_add=None, groups_to_remove=None, @@ -685,13 +684,12 @@ def update_umapi_user(self, umapi_info, user_key, umapi_connector, :type groups_to_remove: set(str) :type umapi_user: dict # with type, username, domain, and email entries """ - is_primary_org = umapi_info.get_name() == PRIMARY_UMAPI_NAME if attributes_to_update or groups_to_add or groups_to_remove: self.updated_user_keys.add(user_key) if attributes_to_update: self.logger.info('Updating info for user key: %s changes: %s', user_key, attributes_to_update) if groups_to_add or groups_to_remove: - if is_primary_org: + if self.is_primary_org(umapi_info): self.logger.info('Managing groups for user key: %s added: %s removed: %s', user_key, groups_to_add, groups_to_remove) else: @@ -705,14 +703,11 @@ def update_umapi_user(self, umapi_info, user_key, umapi_connector, directory_user = umapi_user identity_type = umapi_user.get('type') - commands = self.create_commands_from_directory_user(directory_user, identity_type=identity_type) - if identity_type != user_sync.identity_type.ADOBEID_IDENTITY_TYPE: - commands.update_user(attributes_to_update) - else: - if attributes_to_update: - self.logger.warning("Can't update attributes on Adobe ID user: %s", umapi_user.get("email")) - commands.add_groups(groups_to_add) + commands = user_sync.connector.umapi.Commands(identity_type, directory_user['email'], + directory_user['username'], directory_user['domain']) + commands.update_user(attributes_to_update) commands.remove_groups(groups_to_remove) + commands.add_groups(groups_to_add) umapi_connector.send_commands(commands) def update_umapi_users_for_connector(self, umapi_info, umapi_connector): @@ -731,24 +726,19 @@ def update_umapi_users_for_connector(self, umapi_info, umapi_connector): # the way we construct the return vaue is to start with a map from all directory users # to their groups in this umapi, make a copy, and pop off any adobe users we find. - # That way, and key/value pairs left in the map are the unmatched adobe users and their groups. + # That way, any key/value pairs left in the map are the unmatched adobe users and their groups. user_to_group_map = umapi_info.get_desired_groups_by_user_key() user_to_group_map = {} if user_to_group_map is None else user_to_group_map.copy() - # check to see if we should update adobe users - options = self.options - update_user_info = options['update_user_info'] + # compute all static options before looping over users + in_primary_org = self.is_primary_org(umapi_info) + update_user_info = self.will_update_user_info(umapi_info) manage_groups = self.will_manage_groups() - exclude_strays = self.options['exclude_strays'] - will_process_strays = self.will_process_strays # prepare the strays map if we are going to be processing them - if will_process_strays: + if self.will_process_strays: self.add_stray(umapi_info.get_name(), None) - # there are certain operations we only do in the primary umapi - in_primary_org = umapi_info.get_name() == PRIMARY_UMAPI_NAME - # Walk all the adobe users, getting their group data, matching them with directory users, # and adjusting their attribute and group data accordingly. for umapi_user in umapi_connector.iter_users(): @@ -778,10 +768,10 @@ def update_umapi_users_for_connector(self, umapi_info, umapi_connector): # There's no selected directory user matching this adobe user # so we mark this adobe user as a stray, and we mark him # for removal from any mapped groups. - if exclude_strays: + if self.exclude_strays: self.logger.debug("Excluding Adobe-only user: %s", user_key) self.excluded_user_count += 1 - elif will_process_strays: + elif self.will_process_strays: self.logger.debug("Found Adobe-only user: %s", user_key) self.add_stray(umapi_info.get_name(), user_key, None if not manage_groups else current_groups & umapi_info.get_mapped_groups()) @@ -791,7 +781,7 @@ def update_umapi_users_for_connector(self, umapi_info, umapi_connector): # and mark him for addition and removal of the appropriate mapped groups if update_user_info or manage_groups: self.logger.debug("Adobe user matched on customer side: %s", user_key) - if update_user_info and in_primary_org: + if update_user_info: attribute_differences = self.get_user_attribute_difference(directory_user, umapi_user) if manage_groups: groups_to_add = desired_groups - current_groups @@ -807,7 +797,7 @@ def update_umapi_users_for_connector(self, umapi_info, umapi_connector): def is_umapi_user_excluded(self, in_primary_org, user_key, current_groups): if in_primary_org: - self.adobe_user_count += 1 + self.primary_user_count += 1 # in the primary umapi, we actually check the exclusion conditions identity_type, username, domain = self.parse_user_key(user_key) if identity_type in self.exclude_identity_types: diff --git a/user_sync/version.py b/user_sync/version.py index 6e911ff7c..d203cc8c3 100644 --- a/user_sync/version.py +++ b/user_sync/version.py @@ -18,4 +18,4 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -__version__ = '2.2.1' +__version__ = '2.2.2rc2'