From 73592db9ad1d9021d39a1d1ab5f794a03507ac6f Mon Sep 17 00:00:00 2001 From: Daniel Brotsky Date: Wed, 11 Oct 2017 13:25:21 -0700 Subject: [PATCH 01/28] tentative directional work on group creation We need an issue to hold the spec for this. --- .../1 user-sync-config.yml | 38 +++++++++++++++---- .../config files - basic/3 connector-ldap.yml | 28 +++++++++----- user_sync/connector/directory_ldap.py | 4 ++ 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/examples/config files - basic/1 user-sync-config.yml b/examples/config files - basic/1 user-sync-config.yml index 6845ffa7c..d4906db4d 100644 --- a/examples/config files - basic/1 user-sync-config.yml +++ b/examples/config files - basic/1 user-sync-config.yml @@ -167,13 +167,10 @@ directory_users: # is specified as a list of entries, each of which has a directory_group # setting (whose value is a single directory group) and an adobe_groups # setting (whose value is a list of 0 or more product configuration and - # user groups). All of the values in the adobe_groups settings must - # match the name of product configurations and user groups which have - # already been created on the Adobe side. (In this example, we pretend - # that "Acrobat DC Pro" is a product configuration and "Copy Editors" - # is a user group that the you have already created. Possibly - # the "Copy Editors" user group has been assigned access to appropriate - # Adobe products, such as InDesign and InCopy.) + # user groups). In this example, imagine that "Acrobat DC Pro" is a + # product configuration and "Copy Editors" is a user group, and that + # the "Copy Editors" user group will be assigned access to appropriate + # Adobe products, such as InDesign and InCopy. # [You will need to edit or remove these examples.] - directory_group: "Finance" adobe_groups: @@ -186,6 +183,33 @@ directory_users: - "Copy Editors" - "Acrobat DC Pro" + # (optional) additional_groups (no default value) + # People who use their directory groups for ACLs on the Adobe side + # often have a very large number of groups that they want mapped + # over to (user) groups on the Adobe side. To avoid having to + # specify those groups statically in their config file, and to + # update their config file when they change, they can instead + # use a naming convention for the groups and specify that here. + # The value of this attribute is a mapping from Python regular expressions + # that specify directory groups of interest to Pythonn replacement expressions + # that specify how to construct the name of the target Adobe group + # that the directory group should be mapped to. If a value is + # provided, then all users who are (directly) in groups whose + # CN matches one of the source regular expressions will be put in a user group + # on the Adobe side whose name is given by the target replacement expression. + # The simple example here (which should be removed) maps all the + # groups that start with "ACL-" or end with "-ACL" to an Adobe + # group that starts with "ACL-Grp-". + # (All of these regular expressions must match the entire group name. + # For details on Python regular expression matching and replacement, + # see https://docs.python.org/howto/regex.html ) + additional_groups: + - source: "ACL-(.+)" + target: "ACL-Grp-(\1)" + - source: "(.+)-ACL" + target: "ACL-Grp-(\1)" + + # The limits section provides processing limits which can help ensure that # User Sync jobs do not exceed expected guardrails in their operation limits: diff --git a/examples/config files - basic/3 connector-ldap.yml b/examples/config files - basic/3 connector-ldap.yml index 1021bb813..2d68bd3fa 100755 --- a/examples/config files - basic/3 connector-ldap.yml +++ b/examples/config files - basic/3 connector-ldap.yml @@ -72,7 +72,7 @@ all_users_filter: "(&(objectClass=user)(objectCategory=person)(!(userAccountCont group_filter_format: "(&(|(objectCategory=group)(objectClass=groupOfNames)(objectClass=posixGroup))(cn={group}))" # (optional) group_member_filter_format (default value given below) -# group_users_filter specifies the query used to find all members of a group, +# group_member_filter_format specifies the query used to find all members of a group, # where the string {group_dn} is replaced with the group distinguished name. # The default value just finds users who are immediate members of the group, # not those who are "indirectly" members by virtue of membership in a group @@ -80,6 +80,21 @@ group_filter_format: "(&(|(objectCategory=group)(objectClass=groupOfNames)(objec # use this value instead of the default: # group_member_filter_format: "(memberOf:1.2.840.113556.1.4.1941:={group_dn})" group_member_filter_format: "(memberOf={group_dn})" +# Note that this filter is &-combined with the all_users_filter so that +# only users that would be selected by that filter will be returned as +# members of the given group. + +# (optional) member_group_filter_format (default value given below) +# member_group_filter_format specifies the query used to find all groups that +# directly contain a given member. The string {member_dn} is replaced +# with the DN of the group member. The string {member_uid) is replaced with +# the uid attribute of the group member, if any. The default value expects +# groups to refer to members by their DN. For groups that refer to their +# members by their UID (e.g., posix groups in many OpenLDAP systems), you +# probably want to use this value instead: "(memberUid={member_uid})" +member_group_filter_format: "(member={member_dn})" +# Note that this filter is &-combined with the group_filter_format query +# specifying a wildcard for the group name. So it will only find groups. # (optional) string_encoding (default value given below) # string_encoding specifies the Unicode string encoding used by the directory. @@ -172,14 +187,9 @@ user_email_format: "{mail}" # are already pre-defined attribute names that are used for these fields: # - the Adobe first name is set from the LDAP "givenName" attribute # - the Adobe last name is set from the LDAP "sn" (surname) attribute -# - the Adobe country is set from the LDAP "country" attribute +# - the Adobe country is set from the LDAP "c" (country) attribute # If you need to override these values on the Adobe side, you can use the # custom extension mechanism (see the docs) to compute and set field values -# by combining these and any other custom attributes needed. Seed the +# by combining these and any other custom attributes needed. See the # User Sync documentation for full details. -# -# Finally, some LDAP systems use uids to identify groups, and place users in -# groups via uid rather than name. The User Sync implementation always reads -# the uid attribute on all objects if the directory provides one, so it is -# able to handle directories which function in this way even though the -# configuration files always specify groups by name. + diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index f30acebde..b998aa0b9 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -289,6 +289,10 @@ def iter_users(self, users_filter, extended_attributes): elif last_attribute_name: self.logger.warning('No country code attribute (%s) for user with dn: %s', last_attribute_name, dn) + uid_value = LDAPValueFormatter.get_attribute_value(record, six.text_type('uid')) + source_attributes['uid'] = uid_value + user['member_groups'] = find_member_groups(dn, uid_value if uid_value else six.text_type('')) + if extended_attributes is not None: for extended_attribute in extended_attributes: extended_attribute_value = LDAPValueFormatter.get_attribute_value(record, extended_attribute) From ce76d86e59297ffa4ddf41e46e71d43b6e19637e Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Thu, 2 Nov 2017 11:18:21 -0600 Subject: [PATCH 02/28] comment out optional config options --- examples/config files - basic/1 user-sync-config.yml | 10 +++++----- examples/config files - basic/3 connector-ldap.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/config files - basic/1 user-sync-config.yml b/examples/config files - basic/1 user-sync-config.yml index d4906db4d..ead5d271c 100644 --- a/examples/config files - basic/1 user-sync-config.yml +++ b/examples/config files - basic/1 user-sync-config.yml @@ -203,11 +203,11 @@ directory_users: # (All of these regular expressions must match the entire group name. # For details on Python regular expression matching and replacement, # see https://docs.python.org/howto/regex.html ) - additional_groups: - - source: "ACL-(.+)" - target: "ACL-Grp-(\1)" - - source: "(.+)-ACL" - target: "ACL-Grp-(\1)" + # additional_groups: + # - source: "ACL-(.+)" + # target: "ACL-Grp-(\1)" + # - source: "(.+)-ACL" + # target: "ACL-Grp-(\1)" # The limits section provides processing limits which can help ensure that diff --git a/examples/config files - basic/3 connector-ldap.yml b/examples/config files - basic/3 connector-ldap.yml index 2d68bd3fa..32469bd2e 100755 --- a/examples/config files - basic/3 connector-ldap.yml +++ b/examples/config files - basic/3 connector-ldap.yml @@ -92,7 +92,7 @@ group_member_filter_format: "(memberOf={group_dn})" # groups to refer to members by their DN. For groups that refer to their # members by their UID (e.g., posix groups in many OpenLDAP systems), you # probably want to use this value instead: "(memberUid={member_uid})" -member_group_filter_format: "(member={member_dn})" +# member_group_filter_format: "(member={member_dn})" # Note that this filter is &-combined with the group_filter_format query # specifying a wildcard for the group name. So it will only find groups. From cb82a0ab513ff38b37c70830faf849f458816dc7 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Fri, 3 Nov 2017 08:59:59 -0700 Subject: [PATCH 03/28] query additional direct-membership groups and filter them --- user_sync/app.py | 6 ++++++ user_sync/config.py | 7 +++++++ user_sync/connector/directory_ldap.py | 25 ++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/user_sync/app.py b/user_sync/app.py index 5e7845df5..5dbff9c75 100644 --- a/user_sync/app.py +++ b/user_sync/app.py @@ -307,6 +307,12 @@ def begin_work(config_loader): directory_connector_options['user_identity_type'] = rule_config['new_account_type'] directory_connector.initialize(directory_connector_options) + additional_group_filters = None + if rule_config['directory_additional_groups'] and isinstance(rule_config['directory_additional_groups'], list): + additional_group_filters = [r['source'] for r in rule_config['directory_additional_groups']] + + directory_connector.state.additional_group_filters = additional_group_filters + primary_name = '.primary' if secondary_umapi_configs else '' umapi_primary_connector = user_sync.connector.umapi.UmapiConnector(primary_name, primary_umapi_config) umapi_other_connectors = {} diff --git a/user_sync/config.py b/user_sync/config.py index 9f424dc8a..986f8005e 100644 --- a/user_sync/config.py +++ b/user_sync/config.py @@ -444,6 +444,7 @@ def get_rule_options(self): options.update(self.invocation_options) # process directory configuration options + new_account_type = None directory_config = self.main_config.get_dict_config('directory_users', True) if directory_config: # account type @@ -457,6 +458,12 @@ def get_rule_options(self): default_country_code = directory_config.get_string('default_country_code', True) if default_country_code: options['default_country_code'] = default_country_code + additional_groups = directory_config.get_list('additional_groups', True) or [] + additional_groups = [{'source': re.compile(r['source']), 'target': r['target']} for r in additional_groups] + options['additional_groups'] = additional_groups + if not new_account_type: + new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE + self.logger.debug("Using default for new_account_type: %s", new_account_type) # process exclusion configuration options adobe_config = self.main_config.get_dict_config('adobe_users', True) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index b998aa0b9..a51a5e13c 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -70,6 +70,7 @@ def __init__(self, caller_options): '(&(objectClass=user)(objectCategory=person)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))')) builder.set_string_value('group_member_filter_format', six.text_type( '(memberOf={group_dn})')) + builder.set_string_value('member_group_filter_format', None) builder.set_bool_value('require_tls_cert', False) builder.set_string_value('string_encoding', 'utf8') builder.set_string_value('user_identity_type_format', None) @@ -121,6 +122,7 @@ def __init__(self, caller_options): self.connection = connection logger.debug('Connected') self.user_by_dn = {} + self.additional_group_filters = None def load_users_and_groups(self, groups, extended_attributes, all_users): """ @@ -291,7 +293,14 @@ def iter_users(self, users_filter, extended_attributes): uid_value = LDAPValueFormatter.get_attribute_value(record, six.text_type('uid')) source_attributes['uid'] = uid_value - user['member_groups'] = find_member_groups(dn, uid_value if uid_value else six.text_type('')) + + if self.additional_group_filters: + member_groups = [] + for f in self.additional_group_filters: + for g in self.find_member_groups(dn, uid_value if uid_value else six.text_type('')): + if f.match(g) and g not in member_groups: + member_groups.append(g) + user['member_groups'] = member_groups if extended_attributes is not None: for extended_attribute in extended_attributes: @@ -305,6 +314,20 @@ def iter_users(self, users_filter, extended_attributes): yield (dn, user) + def find_member_groups(self, dn, uid): + member_filter = self.format_ldap_query_string(self.options['member_group_filter_format'], + member_dn=dn, + member_uid=uid) + base_dn = six.text_type(self.options['base_dn']) + res = self.connection.search_s(base_dn, ldap.SCOPE_SUBTREE, + filterstr=member_filter, attrlist=['cn']) + member_groups = [] + for _, group_rec in res: + if isinstance(group_rec, dict): + group_cn = LDAPValueFormatter.get_attribute_value(group_rec, 'cn', first_only=True) + member_groups.append(group_cn) + return member_groups + def iter_search_result(self, base_dn, scope, filter_string, attributes): """ type: filter_string: str From 60e0872c76e2acb6fe1895fad33e79262c698116 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Fri, 3 Nov 2017 13:59:36 -0600 Subject: [PATCH 04/28] rename groups according to additional_groups settings and add them to target groups --- examples/config files - basic/1 user-sync-config.yml | 2 +- user_sync/rules.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/config files - basic/1 user-sync-config.yml b/examples/config files - basic/1 user-sync-config.yml index ead5d271c..90471466d 100644 --- a/examples/config files - basic/1 user-sync-config.yml +++ b/examples/config files - basic/1 user-sync-config.yml @@ -191,7 +191,7 @@ directory_users: # update their config file when they change, they can instead # use a naming convention for the groups and specify that here. # The value of this attribute is a mapping from Python regular expressions - # that specify directory groups of interest to Pythonn replacement expressions + # that specify directory groups of interest to Python replacement expressions # that specify how to construct the name of the target Adobe group # that the directory group should be mapped to. If a value is # provided, then all users who are (directly) in groups whose diff --git a/user_sync/rules.py b/user_sync/rules.py index 0c5248181..0d5dd5f81 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -385,6 +385,13 @@ def read_desired_user_groups(self, mappings, directory_connector): else: self.logger.error('Target adobe group %s is not known; ignored', target_group_qualified_name) + for member_group in directory_user['member_groups']: + for group_rule in self.options['directory_additional_groups']: + if group_rule['source'].match(member_group): + rename_group = group_rule['source'].sub(group_rule['target'], member_group) + for umapi_name, umapi_info in six.iteritems(self.umapi_info_by_name): + umapi_info.add_desired_group_for(user_key, rename_group) + self.logger.debug('Total directory users after filtering: %d', len(filtered_directory_user_by_user_key)) if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('Group work list: %s', dict([(umapi_name, umapi_info.get_desired_groups_by_user_key()) From cabc73d89b6756d47158a2c2056b9fe1b4fba0e2 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Fri, 3 Nov 2017 14:29:52 -0600 Subject: [PATCH 05/28] initialize member_groups --- user_sync/connector/directory_ldap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index a51a5e13c..ab8688db7 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -294,6 +294,7 @@ def iter_users(self, users_filter, extended_attributes): uid_value = LDAPValueFormatter.get_attribute_value(record, six.text_type('uid')) source_attributes['uid'] = uid_value + user['member_groups'] = [] if self.additional_group_filters: member_groups = [] for f in self.additional_group_filters: From 20f2587d3cb9ef76c2939e7d2cd9f4dc0c8520b0 Mon Sep 17 00:00:00 2001 From: Kevin Bhunut Date: Tue, 14 Nov 2017 08:33:37 -0800 Subject: [PATCH 06/28] fixed byte mode error issue in py2 for member_group_filter_format --- user_sync/connector/directory_ldap.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index ab8688db7..a7f960dfa 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -316,12 +316,13 @@ def iter_users(self, users_filter, extended_attributes): yield (dn, user) def find_member_groups(self, dn, uid): - member_filter = self.format_ldap_query_string(self.options['member_group_filter_format'], + member_group_filter_format = six.text_type(self.options['member_group_filter_format']) + member_filter = self.format_ldap_query_string(member_group_filter_format, member_dn=dn, member_uid=uid) base_dn = six.text_type(self.options['base_dn']) res = self.connection.search_s(base_dn, ldap.SCOPE_SUBTREE, - filterstr=member_filter, attrlist=['cn']) + filterstr=member_filter) member_groups = [] for _, group_rec in res: if isinstance(group_rec, dict): From 74b0eca89eb18b43cfd6a36c2387173a0bbdaa2a Mon Sep 17 00:00:00 2001 From: Kevin Bhunut Date: Tue, 14 Nov 2017 10:50:18 -0800 Subject: [PATCH 07/28] Added Auto User-Group Creation feature --- user_sync/config.py | 3 +++ user_sync/connector/umapi.py | 14 ++++++++++++++ user_sync/rules.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/user_sync/config.py b/user_sync/config.py index 986f8005e..6d9b2881e 100644 --- a/user_sync/config.py +++ b/user_sync/config.py @@ -461,6 +461,9 @@ def get_rule_options(self): additional_groups = directory_config.get_list('additional_groups', True) or [] additional_groups = [{'source': re.compile(r['source']), 'target': r['target']} for r in additional_groups] options['additional_groups'] = additional_groups + sync_options = directory_config.get_dict_config('groups_sync_options', True) + if sync_options: + options['auto_create'] = sync_options.get_bool('auto_create', True) if not new_account_type: new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE self.logger.debug("Using default for new_account_type: %s", new_account_type) diff --git a/user_sync/connector/umapi.py b/user_sync/connector/umapi.py index cd86d9676..e7f3cd1ad 100644 --- a/user_sync/connector/umapi.py +++ b/user_sync/connector/umapi.py @@ -146,6 +146,20 @@ def iter_users(self): except umapi_client.UnavailableError as e: raise AssertionException("Error contacting UMAPI server: %s" % e) + def get_groups(self): + return list(self.iter_groups()) + + def iter_groups(self): + try: + for g in umapi_client.UserGroupsQuery(self.connection): + yield g + except umapi_client.UnavailableError as e: + raise AssertionException("Error contacting UMAPI server: %s" % e) + + def create_group(self,name): + if name: + ug = umapi_client.UserGroups(self.connection) + ug.create(name, 'Automatically created by User Sync Tool') def get_action_manager(self): return self.action_manager diff --git a/user_sync/rules.py b/user_sync/rules.py index 0d5dd5f81..0ff168a74 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -174,6 +174,8 @@ def run(self, directory_groups, directory_connector, umapi_connectors): umapi_stats = JobStats('Push to UMAPI' if self.push_umapi else 'Sync with UMAPI', divider="-") umapi_stats.log_start(logger) if directory_connector is not None: + if self.options['auto_create']: + self.sync_umapi_groups(umapi_connectors) self.sync_umapi_users(umapi_connectors) if self.will_process_strays: self.process_strays(umapi_connectors) @@ -389,6 +391,7 @@ def read_desired_user_groups(self, mappings, directory_connector): for group_rule in self.options['directory_additional_groups']: if group_rule['source'].match(member_group): rename_group = group_rule['source'].sub(group_rule['target'], member_group) + umapi_info.add_mapped_group(rename_group) for umapi_name, umapi_info in six.iteritems(self.umapi_info_by_name): umapi_info.add_desired_group_for(user_key, rename_group) @@ -459,6 +462,27 @@ def sync_umapi_users(self, umapi_connectors): self.updated_user_keys.add(user_key) self.create_umapi_user(user_key, groups_to_add, umapi_info, umapi_connector) + def sync_umapi_groups(self, umapi_connectors): + """ + This is where we do sync for user-groups. If auto_create or auto_delete enabled, + this will pull user-groups from console and compare with mapped_groups. If mapped group does exist + in the console, then it will create + :type umapi_connectors: UmapiConnectors + """ + umapi_info, umapi_connector = self.get_umapi_info(PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() + mapped_groups = umapi_info.get_non_normalize_mapped_groups() + #pull all user groups from console + on_adobe_user_groups = [normalize_string(group['name']) for group in umapi_connector.get_groups()] + #verify if group exist + for mapped_group in mapped_groups: + if not normalize_string(mapped_group) in on_adobe_user_groups: + self.logger.info("Auto create user-group enabled: Creating %s" % mapped_group) + try: + # create group + umapi_connector.create_group(mapped_group) + except Exception as e: + self.logger.critical("Unable to create %s user group: %s" % (mapped_group, e)) + def is_selected_user_key(self, user_key): """ :type user_key: str @@ -1154,6 +1178,7 @@ def __init__(self, name): """ self.name = name self.mapped_groups = set() + self.non_normalize_mapped_groups = set() self.desired_groups_by_user_key = {} self.umapi_user_by_user_key = {} self.umapi_users_loaded = False @@ -1170,10 +1195,14 @@ def add_mapped_group(self, group): """ normalized_group_name = normalize_string(group) self.mapped_groups.add(normalized_group_name) + self.non_normalize_mapped_groups.add(group) def get_mapped_groups(self): return self.mapped_groups + def get_non_normalize_mapped_groups(self): + return self.non_normalize_mapped_groups + def get_desired_groups_by_user_key(self): return self.desired_groups_by_user_key From 2f3ea47a1d627a850933935274f5d42e1456bf81 Mon Sep 17 00:00:00 2001 From: Kevin Bhunut Date: Mon, 20 Nov 2017 10:02:36 -0800 Subject: [PATCH 08/28] Resolved #44 - pull both PLC and User-Group --- user_sync/connector/umapi.py | 2 +- user_sync/rules.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/user_sync/connector/umapi.py b/user_sync/connector/umapi.py index e7f3cd1ad..e40bbdf8f 100644 --- a/user_sync/connector/umapi.py +++ b/user_sync/connector/umapi.py @@ -151,7 +151,7 @@ def get_groups(self): def iter_groups(self): try: - for g in umapi_client.UserGroupsQuery(self.connection): + for g in umapi_client.GroupsQuery(self.connection): yield g except umapi_client.UnavailableError as e: raise AssertionException("Error contacting UMAPI server: %s" % e) diff --git a/user_sync/rules.py b/user_sync/rules.py index 0ff168a74..5a49da573 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -472,10 +472,10 @@ def sync_umapi_groups(self, umapi_connectors): umapi_info, umapi_connector = self.get_umapi_info(PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() mapped_groups = umapi_info.get_non_normalize_mapped_groups() #pull all user groups from console - on_adobe_user_groups = [normalize_string(group['name']) for group in umapi_connector.get_groups()] + on_adobe_groups = [normalize_string(group['groupName']) for group in umapi_connector.get_groups()] #verify if group exist for mapped_group in mapped_groups: - if not normalize_string(mapped_group) in on_adobe_user_groups: + if not normalize_string(mapped_group) in on_adobe_groups: self.logger.info("Auto create user-group enabled: Creating %s" % mapped_group) try: # create group From fcb04aca35e76540940aa2723054c6c2817b6474 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Mon, 20 Nov 2017 13:13:23 -0700 Subject: [PATCH 09/28] group_sync_options in example --- .../config files - basic/1 user-sync-config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/examples/config files - basic/1 user-sync-config.yml b/examples/config files - basic/1 user-sync-config.yml index 90471466d..fa09a2e5d 100644 --- a/examples/config files - basic/1 user-sync-config.yml +++ b/examples/config files - basic/1 user-sync-config.yml @@ -209,6 +209,19 @@ directory_users: # - source: "(.+)-ACL" # target: "ACL-Grp-(\1)" + # (optional) group_sync_options (default: all options false) + # Options that govern the automatic creation and/or deletion of Adobe user groups + # auto_create: + # Automatically create target groups that do not exist. Non-existent groups + # will *always* be created as Adobe groups. If targeting product profiles, they + # must always be created manually in the Admin Console + # auto_delete: + # Automatically delete user groups that don't have any members at the time of sync. + # The only groups considered for deletion are groups mapped in the renaming rules + # specified in additional_groups. + # group_sync_options: + # auto_create: False + # auto_delete: False # The limits section provides processing limits which can help ensure that # User Sync jobs do not exceed expected guardrails in their operation From 675bfe7c9ebba88b62114fe514c3eef0fd8b8ee7 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Fri, 3 Nov 2017 08:59:59 -0700 Subject: [PATCH 10/28] query additional direct-membership groups and filter them --- user_sync/connector/directory_ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index a7f960dfa..52f6906fd 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -322,7 +322,7 @@ def find_member_groups(self, dn, uid): member_uid=uid) base_dn = six.text_type(self.options['base_dn']) res = self.connection.search_s(base_dn, ldap.SCOPE_SUBTREE, - filterstr=member_filter) + filterstr=member_filter, attrlist=['cn']) member_groups = [] for _, group_rec in res: if isinstance(group_rec, dict): From 7379fe3b7e3fce223613f5daef2946da37f1e701 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Fri, 3 Nov 2017 13:59:36 -0600 Subject: [PATCH 11/28] rename groups according to additional_groups settings and add them to target groups --- user_sync/rules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/user_sync/rules.py b/user_sync/rules.py index 5a49da573..907f9fcc5 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -395,6 +395,7 @@ def read_desired_user_groups(self, mappings, directory_connector): for umapi_name, umapi_info in six.iteritems(self.umapi_info_by_name): umapi_info.add_desired_group_for(user_key, rename_group) + self.logger.debug('Total directory users after filtering: %d', len(filtered_directory_user_by_user_key)) if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('Group work list: %s', dict([(umapi_name, umapi_info.get_desired_groups_by_user_key()) From 997fd1a8c7aa93a843b53e20bdcfe0e56c495735 Mon Sep 17 00:00:00 2001 From: Kevin Bhunut Date: Tue, 14 Nov 2017 08:33:37 -0800 Subject: [PATCH 12/28] fixed byte mode error issue in py2 for member_group_filter_format --- user_sync/connector/directory_ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index 52f6906fd..a7f960dfa 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -322,7 +322,7 @@ def find_member_groups(self, dn, uid): member_uid=uid) base_dn = six.text_type(self.options['base_dn']) res = self.connection.search_s(base_dn, ldap.SCOPE_SUBTREE, - filterstr=member_filter, attrlist=['cn']) + filterstr=member_filter) member_groups = [] for _, group_rec in res: if isinstance(group_rec, dict): From 9b23000edd9ff0ebb1df38b89c6f9f09c7e2ab62 Mon Sep 17 00:00:00 2001 From: Kevin Bhunut Date: Tue, 14 Nov 2017 10:50:18 -0800 Subject: [PATCH 13/28] Added Auto User-Group Creation feature --- user_sync/config.py | 3 +++ user_sync/connector/umapi.py | 1 + 2 files changed, 4 insertions(+) diff --git a/user_sync/config.py b/user_sync/config.py index 6d9b2881e..e51682934 100644 --- a/user_sync/config.py +++ b/user_sync/config.py @@ -464,6 +464,9 @@ def get_rule_options(self): sync_options = directory_config.get_dict_config('groups_sync_options', True) if sync_options: options['auto_create'] = sync_options.get_bool('auto_create', True) + sync_options = directory_config.get_dict_config('groups_sync_options', True) + if sync_options: + options['auto_create'] = sync_options.get_bool('auto_create', True) if not new_account_type: new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE self.logger.debug("Using default for new_account_type: %s", new_account_type) diff --git a/user_sync/connector/umapi.py b/user_sync/connector/umapi.py index e40bbdf8f..6c7916e60 100644 --- a/user_sync/connector/umapi.py +++ b/user_sync/connector/umapi.py @@ -160,6 +160,7 @@ def create_group(self,name): if name: ug = umapi_client.UserGroups(self.connection) ug.create(name, 'Automatically created by User Sync Tool') + def get_action_manager(self): return self.action_manager From ac17786d209747542890b84d47c21a951ca6b580 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Tue, 21 Nov 2017 13:27:43 -0700 Subject: [PATCH 14/28] update group_sync_options config key to match spec --- user_sync/config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/user_sync/config.py b/user_sync/config.py index e51682934..604e3ade2 100644 --- a/user_sync/config.py +++ b/user_sync/config.py @@ -461,10 +461,7 @@ def get_rule_options(self): additional_groups = directory_config.get_list('additional_groups', True) or [] additional_groups = [{'source': re.compile(r['source']), 'target': r['target']} for r in additional_groups] options['additional_groups'] = additional_groups - sync_options = directory_config.get_dict_config('groups_sync_options', True) - if sync_options: - options['auto_create'] = sync_options.get_bool('auto_create', True) - sync_options = directory_config.get_dict_config('groups_sync_options', True) + sync_options = directory_config.get_dict_config('group_sync_options', True) if sync_options: options['auto_create'] = sync_options.get_bool('auto_create', True) if not new_account_type: From 5caaecdfa368e3b7b67172d6e2725c3878a86d5d Mon Sep 17 00:00:00 2001 From: Kevin Bhunut Date: Mon, 4 Dec 2017 11:12:17 -0800 Subject: [PATCH 15/28] Add Auto Delete feature --- user_sync/connector/umapi.py | 10 ++++++ user_sync/rules.py | 67 +++++++++++++++++++++++++++--------- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/user_sync/connector/umapi.py b/user_sync/connector/umapi.py index 6c7916e60..3056c71e9 100644 --- a/user_sync/connector/umapi.py +++ b/user_sync/connector/umapi.py @@ -156,6 +156,16 @@ def iter_groups(self): except umapi_client.UnavailableError as e: raise AssertionException("Error contacting UMAPI server: %s" % e) + def get_user_groups(self): + return list(self.iter_user_groups()) + + def iter_user_groups(self): + try: + for g in umapi_client.UserGroupsQuery(self.connection): + yield g + except umapi_client.UnavailableError as e: + raise AssertionException("Error contacting UMAPI server: %s" % e) + def create_group(self,name): if name: ug = umapi_client.UserGroups(self.connection) diff --git a/user_sync/rules.py b/user_sync/rules.py index 907f9fcc5..91c5544fd 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -21,6 +21,7 @@ import logging import six +import re import user_sync.connector.umapi import user_sync.error @@ -70,6 +71,8 @@ def __init__(self, caller_options): # counters for action summary log self.action_summary = { # these are in alphabetical order! Always add new ones that way! + 'adobe_user_groups_created': 0, + 'adobe_user_groups_deleted': 0, 'directory_users_read': 0, 'directory_users_selected': 0, 'excluded_user_count': 0, @@ -174,11 +177,11 @@ def run(self, directory_groups, directory_connector, umapi_connectors): umapi_stats = JobStats('Push to UMAPI' if self.push_umapi else 'Sync with UMAPI', divider="-") umapi_stats.log_start(logger) if directory_connector is not None: - if self.options['auto_create']: - self.sync_umapi_groups(umapi_connectors) + self.create_umapi_groups(umapi_connectors) self.sync_umapi_users(umapi_connectors) if self.will_process_strays: self.process_strays(umapi_connectors) + self.delete_umapi_groups(umapi_connectors) umapi_connectors.execute_actions() umapi_stats.log_end(logger) self.log_action_summary(umapi_connectors) @@ -234,6 +237,8 @@ def log_action_summary(self, umapi_connectors): ['unchanged_user_count', 'Number of non-excluded Adobe users with no changes'], ['primary_users_created', 'Number of new Adobe users added'], ['updated_user_count', 'Number of matching Adobe users updated'], + ['adobe_user_groups_created', 'Number of Adobe user-groups created'], + ['adobe_user_groups_deleted', 'Number of Adobe user-groups deleted'], ] if umapi_connectors.get_secondary_connectors(): action_summary_description += [ @@ -463,26 +468,56 @@ def sync_umapi_users(self, umapi_connectors): self.updated_user_keys.add(user_key) self.create_umapi_user(user_key, groups_to_add, umapi_info, umapi_connector) - def sync_umapi_groups(self, umapi_connectors): + def create_umapi_groups(self, umapi_connectors): """ - This is where we do sync for user-groups. If auto_create or auto_delete enabled, + This is where we create user-groups. If auto_create is enabled, this will pull user-groups from console and compare with mapped_groups. If mapped group does exist - in the console, then it will create + in the console, then it will create. Note: Push Mode is not supported :type umapi_connectors: UmapiConnectors """ - umapi_info, umapi_connector = self.get_umapi_info(PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() - mapped_groups = umapi_info.get_non_normalize_mapped_groups() - #pull all user groups from console - on_adobe_groups = [normalize_string(group['groupName']) for group in umapi_connector.get_groups()] - #verify if group exist - for mapped_group in mapped_groups: - if not normalize_string(mapped_group) in on_adobe_groups: - self.logger.info("Auto create user-group enabled: Creating %s" % mapped_group) + if not self.push_umapi: + umapi_info, umapi_connector = self.get_umapi_info( + PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() + mapped_groups = umapi_info.get_non_normalize_mapped_groups() + # pull all user groups from console + on_adobe_groups = umapi_connector.get_groups() + # verify if group exist and create + if self.options['auto_create']: + for mapped_group in mapped_groups: + if not filter(lambda grp: normalize_string(grp['groupName']) == normalize_string(mapped_group), + on_adobe_groups): + self.logger.info("Auto create user-group enabled: Creating %s" % mapped_group) + try: + # create group + umapi_connector.create_group(mapped_group) + self.action_summary['adobe_user_groups_created'] += 1 + except Exception as e: + self.logger.critical("Unable to create %s user group: %s" % (mapped_group, e)) + + def delete_umapi_groups(self, umapi_connectors): + """ + This is where we delete user-groups. If auto_create is enabled, + this will pull user-groups from console, If on adobe user-groups match the + auto_delete_filters and group member is 0 then it will delete. Note: Push Mode is not supported + :type umapi_connectors: UmapiConnectors + """ + if not self.push_umapi: + umapi_info, umapi_connector = self.get_umapi_info( + PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() + if self.options['auto_delete']: + # retreive auto_delete_filters options + delete_rules = self.options['auto_delete_filters'] or [] + on_adobe_user_groups = umapi_connector.get_user_groups() + for rule in delete_rules: + filtered_groups = [g for g in on_adobe_user_groups if + (('userCount' not in g) and (rule.match(g['name'])))] + for group in filtered_groups: + self.logger.info("Auto Delete user-group enabled: Deleting %s" % group['name']) try: - # create group - umapi_connector.create_group(mapped_group) + umapi_connector.delete_group(group['groupId']) + self.action_summary['adobe_user_groups_deleted'] += 1 except Exception as e: - self.logger.critical("Unable to create %s user group: %s" % (mapped_group, e)) + self.logger.critical("Unable to delete %s user group: %s" % (group['name'], e)) def is_selected_user_key(self, user_key): """ From 54a8ca6589f27105a6263bba2a300d79746fa960 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 21 Feb 2018 13:22:27 -0700 Subject: [PATCH 16/28] umapi-client interface changes --- user_sync/connector/umapi.py | 7 ++++--- user_sync/rules.py | 33 +++------------------------------ 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/user_sync/connector/umapi.py b/user_sync/connector/umapi.py index 3056c71e9..18e27acde 100644 --- a/user_sync/connector/umapi.py +++ b/user_sync/connector/umapi.py @@ -166,10 +166,11 @@ def iter_user_groups(self): except umapi_client.UnavailableError as e: raise AssertionException("Error contacting UMAPI server: %s" % e) - def create_group(self,name): + def create_group(self, name): if name: - ug = umapi_client.UserGroups(self.connection) - ug.create(name, 'Automatically created by User Sync Tool') + group = umapi_client.UserGroupAction(group_name=name) + group.create(description="Automatically created by User Sync Tool") + return self.connection.execute_single(group) def get_action_manager(self): return self.action_manager diff --git a/user_sync/rules.py b/user_sync/rules.py index 91c5544fd..5caaa3c86 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -181,7 +181,6 @@ def run(self, directory_groups, directory_connector, umapi_connectors): self.sync_umapi_users(umapi_connectors) if self.will_process_strays: self.process_strays(umapi_connectors) - self.delete_umapi_groups(umapi_connectors) umapi_connectors.execute_actions() umapi_stats.log_end(logger) self.log_action_summary(umapi_connectors) @@ -480,45 +479,19 @@ def create_umapi_groups(self, umapi_connectors): PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() mapped_groups = umapi_info.get_non_normalize_mapped_groups() # pull all user groups from console - on_adobe_groups = umapi_connector.get_groups() + on_adobe_groups = [normalize_string(g['groupName']) for g in umapi_connector.get_groups()] # verify if group exist and create if self.options['auto_create']: for mapped_group in mapped_groups: - if not filter(lambda grp: normalize_string(grp['groupName']) == normalize_string(mapped_group), - on_adobe_groups): + if normalize_string(mapped_group) not in on_adobe_groups: self.logger.info("Auto create user-group enabled: Creating %s" % mapped_group) try: # create group - umapi_connector.create_group(mapped_group) + res = umapi_connector.create_group(mapped_group) self.action_summary['adobe_user_groups_created'] += 1 except Exception as e: self.logger.critical("Unable to create %s user group: %s" % (mapped_group, e)) - def delete_umapi_groups(self, umapi_connectors): - """ - This is where we delete user-groups. If auto_create is enabled, - this will pull user-groups from console, If on adobe user-groups match the - auto_delete_filters and group member is 0 then it will delete. Note: Push Mode is not supported - :type umapi_connectors: UmapiConnectors - """ - if not self.push_umapi: - umapi_info, umapi_connector = self.get_umapi_info( - PRIMARY_UMAPI_NAME), umapi_connectors.get_primary_connector() - if self.options['auto_delete']: - # retreive auto_delete_filters options - delete_rules = self.options['auto_delete_filters'] or [] - on_adobe_user_groups = umapi_connector.get_user_groups() - for rule in delete_rules: - filtered_groups = [g for g in on_adobe_user_groups if - (('userCount' not in g) and (rule.match(g['name'])))] - for group in filtered_groups: - self.logger.info("Auto Delete user-group enabled: Deleting %s" % group['name']) - try: - umapi_connector.delete_group(group['groupId']) - self.action_summary['adobe_user_groups_deleted'] += 1 - except Exception as e: - self.logger.critical("Unable to delete %s user group: %s" % (group['name'], e)) - def is_selected_user_key(self, user_key): """ :type user_key: str From 141c85a12dd2c377e374b61f39bd75ec86aee881 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Mon, 26 Feb 2018 13:22:01 -0700 Subject: [PATCH 17/28] example config updates --- examples/config files - basic/1 user-sync-config.yml | 10 ++++------ examples/config files - basic/3 connector-ldap.yml | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/config files - basic/1 user-sync-config.yml b/examples/config files - basic/1 user-sync-config.yml index fa09a2e5d..d3dfd46c3 100644 --- a/examples/config files - basic/1 user-sync-config.yml +++ b/examples/config files - basic/1 user-sync-config.yml @@ -214,14 +214,12 @@ directory_users: # auto_create: # Automatically create target groups that do not exist. Non-existent groups # will *always* be created as Adobe groups. If targeting product profiles, they - # must always be created manually in the Admin Console - # auto_delete: - # Automatically delete user groups that don't have any members at the time of sync. - # The only groups considered for deletion are groups mapped in the renaming rules - # specified in additional_groups. + # must always be created manually in the Admin Console. + # NOTE: auto_create applies to any targeted Adobe group that doesn't exist. + # This includes groups targeted through group mapping, extension config, + # and the additional_groups functionality. # group_sync_options: # auto_create: False - # auto_delete: False # The limits section provides processing limits which can help ensure that # User Sync jobs do not exceed expected guardrails in their operation diff --git a/examples/config files - basic/3 connector-ldap.yml b/examples/config files - basic/3 connector-ldap.yml index 32469bd2e..8f13d8eb1 100755 --- a/examples/config files - basic/3 connector-ldap.yml +++ b/examples/config files - basic/3 connector-ldap.yml @@ -92,9 +92,9 @@ group_member_filter_format: "(memberOf={group_dn})" # groups to refer to members by their DN. For groups that refer to their # members by their UID (e.g., posix groups in many OpenLDAP systems), you # probably want to use this value instead: "(memberUid={member_uid})" -# member_group_filter_format: "(member={member_dn})" # Note that this filter is &-combined with the group_filter_format query # specifying a wildcard for the group name. So it will only find groups. +# member_group_filter_format: "(member={member_dn})" # (optional) string_encoding (default value given below) # string_encoding specifies the Unicode string encoding used by the directory. From 008a6cf39ac4e1cc235330dd5f7c1fb782fdf6d9 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Tue, 27 Feb 2018 16:00:36 -0700 Subject: [PATCH 18/28] add group sync docs --- docs/en/user-manual/advanced_configuration.md | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/docs/en/user-manual/advanced_configuration.md b/docs/en/user-manual/advanced_configuration.md index 369c7f38c..e47feb0a6 100644 --- a/docs/en/user-manual/advanced_configuration.md +++ b/docs/en/user-manual/advanced_configuration.md @@ -681,6 +681,121 @@ In order to use the Okta connector, you will need to specify the `--connector ok Okta sync can use extended groups, attributes and after-mapping hooks. The names of extended attributes must be valid Okta profile fields. +## Additional Group Options + +Some Adobe applications, such as Adobe Experience Manager, may have a +number of permission-based or role-based groups that need to be +represented in the Adobe Admin Console. These groups have no special +purpose in the console, apart from serving as containers for users. +However, because of how different Adobe products integrate with one +another, they must be represented in the console. + +The User Sync Tool can target these groups with its +`member_group_filter_format` and `additional_groups` config options. + +### Member Group Filter Format + +`member_group_filter_format` is defined in `connector-ldap.yml`. +It specifies an LDAP query that identifies the groups that directly +contain a given user. Since a user is likely to belong to LDAP groups +that are not relevant to user sync, groups queried with this mechanism +are filtered with the rules specified in `additional_groups`. + +`member_group_filter_format` is executed once per user, and must +contain a placeholder for some kind of user identifier. + +The following example query returns groups that have a member defined +by `member_dn`. `member_dn` is the distinguished name of an Active +Directory user. Other directory systems may use different +identifiers such as `member_uid`. + +```yaml +member_group_filter_format: "(member={member_dn})" +``` + +### Additional Group Rules + +`additional_groups` is defined in `user-sync-config.yml`. It specifies +a list of rules to identify and filter groups returned in +`member_group_filter_format`, as well as rules that govern how +corresponding Adobe groups should be named. + +For example - suppose an Adobe Experience Manager customer would like +to sync all AEM users to the admin console. They define a group +mapping in `user-sync-config.yml` like the following: + +```yaml + - directory_group: "AEM-USERS" + adobe_groups: + - "Adobe Experience Manager" +``` + +All users that belong to the directory group `AEM-USERS` will be +assigned to the `Adobe Experience Manager` product profile. + +This example company's AEM users fall into two broad categories - +authors and publishers. These users already belong to LDAP groups that +correspond to each role - `AEM-ACL-AUTHORS` and `AEM-ACL-PUBLISHERS`, +respectively. Suppose this company wishes to assign users to these +additional groups when syncing users. Assuming their +`member_group_filter_format` query is configured, they can leverage the +`additional_group` config option: + +```yaml + additional_groups: + - source: "AEM-ACL-(.+)" + target: "AEM-(\1)" +``` + +`additional_groups` contains a list of additional group rules. `source` +is a regular expression that identifies the group. Only groups that +match a `source` regex will be included. `target` is a regex +substitution string that allows group names to be renamed. In this +case, any group beginning with `AEM-ACL` will be renamed to `AEM-[role]`. +Each rule is executed on the list of groups returned by +`member_group_filter_format` for each user. In this example, authors +and publishers are added to their respective Adobe user group +(`AEM-AUTHORS` or `AEM-PUBLISHERS`). + +Note: The company in this example can also add mappings for authors +and publishers to the group mapping in `user-sync-config.yml`. The +key advantage to using the additional groups mechanism is that it +will apply dynamically to any LDAP group that matches the regex +`AEM-ACL-(.+)`. If additional AEM roles are introduced, they will +be included in sync as long as they follow that naming convention - +no configuration change would be needed. + +## Automatic Group Creation + +The User Sync Tool can be configured to automatically create targeted +Adobe user groups that do not already exist. This can be used in +conjunction with the additional groups functionality detailed in the +previous section, but it also applies to Adobe groups targeted in +the group mapping as well as the extension config. + +`group_sync_options` is defined in `user-sync-config.yml`. It contains +an object that currently has just one key - `auto_create`. +`auto_create` is boolean and is `False` by default. + +To enable dynamic group creation, set `auto_create` to `True`: + +```yaml + group_sync_options: + auto_create: True +``` + +With auto create enabled, a given Adobe group will be created if the +following conditions are true: + +1. Group is targeted for at least one user +2. Group does not currently exist + +New groups are always created as user groups. The UMAPI does not +support product profile creation, so the Sync Tool can't create them. +If the Sync Tool is configured to target a misspelled profile name, or +a profile that doesn't exist, it will automatically create a group with +the specified name. + --- [Previous Section](usage_scenarios.md) \| [Next Section](deployment_best_practices.md) From 1ccae679c16714785ca14e633cf7d56d49877087 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Tue, 27 Feb 2018 21:55:56 -0700 Subject: [PATCH 19/28] post-rebase bug fixes --- user_sync/app.py | 5 +++-- user_sync/rules.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/user_sync/app.py b/user_sync/app.py index 5dbff9c75..86b9bb6ae 100644 --- a/user_sync/app.py +++ b/user_sync/app.py @@ -308,8 +308,9 @@ def begin_work(config_loader): directory_connector.initialize(directory_connector_options) additional_group_filters = None - if rule_config['directory_additional_groups'] and isinstance(rule_config['directory_additional_groups'], list): - additional_group_filters = [r['source'] for r in rule_config['directory_additional_groups']] + additional_groups = rule_config.get('additional_groups', None) + if additional_groups and isinstance(additional_groups, list): + additional_group_filters = [r['source'] for r in additional_groups] directory_connector.state.additional_group_filters = additional_group_filters diff --git a/user_sync/rules.py b/user_sync/rules.py index 5caaa3c86..45a6bfb57 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -391,8 +391,9 @@ def read_desired_user_groups(self, mappings, directory_connector): else: self.logger.error('Target adobe group %s is not known; ignored', target_group_qualified_name) + additional_groups = self.options.get('additional_groups', []) for member_group in directory_user['member_groups']: - for group_rule in self.options['directory_additional_groups']: + for group_rule in additional_groups: if group_rule['source'].match(member_group): rename_group = group_rule['source'].sub(group_rule['target'], member_group) umapi_info.add_mapped_group(rename_group) From 81d4c9fcfd3266e24c5dad99e33398185d32c648 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 28 Feb 2018 10:06:32 -0700 Subject: [PATCH 20/28] bump umapi-client dependency version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cfdfef9c6..f634c10df 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ 'pyldap==2.4.45', 'PyYAML', 'six', - 'umapi-client>=2.10', + 'umapi-client>=2.11', ], extras_require={ ':sys_platform=="linux" or sys_platform=="linux2"':[ From e757f3d8d3a70d82877a45bead98fd88fb8eccfe Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 28 Feb 2018 10:21:54 -0700 Subject: [PATCH 21/28] remove remaining references to group deletion --- user_sync/rules.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/user_sync/rules.py b/user_sync/rules.py index 45a6bfb57..845da5832 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -72,7 +72,6 @@ def __init__(self, caller_options): self.action_summary = { # these are in alphabetical order! Always add new ones that way! 'adobe_user_groups_created': 0, - 'adobe_user_groups_deleted': 0, 'directory_users_read': 0, 'directory_users_selected': 0, 'excluded_user_count': 0, @@ -237,7 +236,6 @@ def log_action_summary(self, umapi_connectors): ['primary_users_created', 'Number of new Adobe users added'], ['updated_user_count', 'Number of matching Adobe users updated'], ['adobe_user_groups_created', 'Number of Adobe user-groups created'], - ['adobe_user_groups_deleted', 'Number of Adobe user-groups deleted'], ] if umapi_connectors.get_secondary_connectors(): action_summary_description += [ From d999650b2bce6a1de98a951345b2baa8250e1f14 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 15 Aug 2018 11:39:50 -0600 Subject: [PATCH 22/28] ensure that auto_create is optional --- user_sync/rules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/user_sync/rules.py b/user_sync/rules.py index 845da5832..235dfc5c6 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -480,7 +480,8 @@ def create_umapi_groups(self, umapi_connectors): # pull all user groups from console on_adobe_groups = [normalize_string(g['groupName']) for g in umapi_connector.get_groups()] # verify if group exist and create - if self.options['auto_create']: + auto_create = self.options.get('auto_create', None) + if auto_create: for mapped_group in mapped_groups: if normalize_string(mapped_group) not in on_adobe_groups: self.logger.info("Auto create user-group enabled: Creating %s" % mapped_group) From 3fe53eeab65896fb9dce06416469111bec21a650 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 15 Aug 2018 16:01:58 -0600 Subject: [PATCH 23/28] append memberOf to LDAP query instead of performing additional group membership query --- user_sync/connector/directory_ldap.py | 34 +++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index a7f960dfa..f95f07352 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -28,6 +28,7 @@ import user_sync.error import user_sync.identity_type from user_sync.error import AssertionException +from ldap import dn def connector_metadata(): @@ -211,6 +212,7 @@ def iter_users(self, users_filter, extended_attributes): user_attribute_names.extend(self.user_email_formatter.get_attribute_names()) user_attribute_names.extend(self.user_username_formatter.get_attribute_names()) user_attribute_names.extend(self.user_domain_formatter.get_attribute_names()) + user_attribute_names.append('memberOf') extended_attributes = [six.text_type(attr) for attr in extended_attributes] extended_attributes = list(set(extended_attributes) - set(user_attribute_names)) @@ -298,7 +300,7 @@ def iter_users(self, users_filter, extended_attributes): if self.additional_group_filters: member_groups = [] for f in self.additional_group_filters: - for g in self.find_member_groups(dn, uid_value if uid_value else six.text_type('')): + for g in self.get_member_groups(record): if f.match(g) and g not in member_groups: member_groups.append(g) user['member_groups'] = member_groups @@ -315,20 +317,22 @@ def iter_users(self, users_filter, extended_attributes): yield (dn, user) - def find_member_groups(self, dn, uid): - member_group_filter_format = six.text_type(self.options['member_group_filter_format']) - member_filter = self.format_ldap_query_string(member_group_filter_format, - member_dn=dn, - member_uid=uid) - base_dn = six.text_type(self.options['base_dn']) - res = self.connection.search_s(base_dn, ldap.SCOPE_SUBTREE, - filterstr=member_filter) - member_groups = [] - for _, group_rec in res: - if isinstance(group_rec, dict): - group_cn = LDAPValueFormatter.get_attribute_value(group_rec, 'cn', first_only=True) - member_groups.append(group_cn) - return member_groups + def get_member_groups(self, user): + group_names = [] + groups = LDAPValueFormatter.get_attribute_value(user, 'memberOf') + for group_dn in map(dn.str2dn, groups): + group_cn = self.get_cn_from_dn(group_dn) + if group_cn: + group_names.append(group_cn) + return group_names + + @staticmethod + def get_cn_from_dn(group_dn): + for rdn in group_dn: + for rdn_part in rdn: + if rdn_part[0].lower() == 'cn': + return rdn_part[1] + return None def iter_search_result(self, base_dn, scope, filter_string, attributes): """ From 40b0abe4224eb7a3a69328491c88338637cd8afb Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 15 Aug 2018 16:38:33 -0600 Subject: [PATCH 24/28] add comments to group name parsing --- user_sync/connector/directory_ldap.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index f95f07352..b75ad7849 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -318,6 +318,12 @@ def iter_users(self, users_filter, extended_attributes): yield (dn, user) def get_member_groups(self, user): + """ + Get a list of member group common names for user + Assumes groups are contained in attribute memberOf + :param user: + :return: + """ group_names = [] groups = LDAPValueFormatter.get_attribute_value(user, 'memberOf') for group_dn in map(dn.str2dn, groups): @@ -328,6 +334,13 @@ def get_member_groups(self, user): @staticmethod def get_cn_from_dn(group_dn): + """ + Take a DN parsed by ldap.dn.str2dn and locate and return the common name + Returns None if no common name is found + If common name is complex (e.g. cn=Bob Jones+email=bob.jones@example.com) then first part of CN is returned + :param group_dn: + :return: + """ for rdn in group_dn: for rdn_part in rdn: if rdn_part[0].lower() == 'cn': From 47295a95da1020605e56ed59c01ff8a43b342378 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 15 Aug 2018 17:05:10 -0600 Subject: [PATCH 25/28] remove config option for member_group_filter_format --- docs/en/user-manual/advanced_configuration.md | 55 +++++++------------ .../config files - basic/3 connector-ldap.yml | 12 ---- user_sync/connector/directory_ldap.py | 1 - 3 files changed, 19 insertions(+), 49 deletions(-) diff --git a/docs/en/user-manual/advanced_configuration.md b/docs/en/user-manual/advanced_configuration.md index e47feb0a6..a257ab62d 100644 --- a/docs/en/user-manual/advanced_configuration.md +++ b/docs/en/user-manual/advanced_configuration.md @@ -690,35 +690,15 @@ purpose in the console, apart from serving as containers for users. However, because of how different Adobe products integrate with one another, they must be represented in the console. -The User Sync Tool can target these groups with its -`member_group_filter_format` and `additional_groups` config options. - -### Member Group Filter Format - -`member_group_filter_format` is defined in `connector-ldap.yml`. -It specifies an LDAP query that identifies the groups that directly -contain a given user. Since a user is likely to belong to LDAP groups -that are not relevant to user sync, groups queried with this mechanism -are filtered with the rules specified in `additional_groups`. - -`member_group_filter_format` is executed once per user, and must -contain a placeholder for some kind of user identifier. - -The following example query returns groups that have a member defined -by `member_dn`. `member_dn` is the distinguished name of an Active -Directory user. Other directory systems may use different -identifiers such as `member_uid`. - -```yaml -member_group_filter_format: "(member={member_dn})" -``` +The User Sync Tool can target these groups with the `additional_groups` +config option. ### Additional Group Rules -`additional_groups` is defined in `user-sync-config.yml`. It specifies -a list of rules to identify and filter groups returned in -`member_group_filter_format`, as well as rules that govern how -corresponding Adobe groups should be named. +`additional_groups` is defined in `user-sync-config.yml` in the `groups` +object. It specifies a list of rules to identify and filter groups +present in the `memberOf` LDAP attribute, as well as rules that govern +how corresponding Adobe groups should be named. For example - suppose an Adobe Experience Manager customer would like to sync all AEM users to the admin console. They define a group @@ -737,14 +717,18 @@ This example company's AEM users fall into two broad categories - authors and publishers. These users already belong to LDAP groups that correspond to each role - `AEM-ACL-AUTHORS` and `AEM-ACL-PUBLISHERS`, respectively. Suppose this company wishes to assign users to these -additional groups when syncing users. Assuming their -`member_group_filter_format` query is configured, they can leverage the -`additional_group` config option: +additional groups when syncing users. Assuming group membership +information can be found in the `memberOf` user attribute, they can +leverage the `additional_group` config option: ```yaml - additional_groups: - - source: "AEM-ACL-(.+)" - target: "AEM-(\1)" +directory_users: + # ... additional directory config options + groups: + # ... group mappings, etc + additional_groups: + - source: "AEM-ACL-(.+)" + target: "AEM-(\1)" ``` `additional_groups` contains a list of additional group rules. `source` @@ -752,10 +736,9 @@ is a regular expression that identifies the group. Only groups that match a `source` regex will be included. `target` is a regex substitution string that allows group names to be renamed. In this case, any group beginning with `AEM-ACL` will be renamed to `AEM-[role]`. -Each rule is executed on the list of groups returned by -`member_group_filter_format` for each user. In this example, authors -and publishers are added to their respective Adobe user group -(`AEM-AUTHORS` or `AEM-PUBLISHERS`). +Each rule is executed on the list of groups a user directly belongs to. +In this example, authors and publishers are added to their respective +Adobe user group (`AEM-AUTHORS` or `AEM-PUBLISHERS`). Note: The company in this example can also add mappings for authors and publishers to the group mapping in `user-sync-config.yml`. The diff --git a/examples/config files - basic/3 connector-ldap.yml b/examples/config files - basic/3 connector-ldap.yml index 8f13d8eb1..87cd6c864 100755 --- a/examples/config files - basic/3 connector-ldap.yml +++ b/examples/config files - basic/3 connector-ldap.yml @@ -84,18 +84,6 @@ group_member_filter_format: "(memberOf={group_dn})" # only users that would be selected by that filter will be returned as # members of the given group. -# (optional) member_group_filter_format (default value given below) -# member_group_filter_format specifies the query used to find all groups that -# directly contain a given member. The string {member_dn} is replaced -# with the DN of the group member. The string {member_uid) is replaced with -# the uid attribute of the group member, if any. The default value expects -# groups to refer to members by their DN. For groups that refer to their -# members by their UID (e.g., posix groups in many OpenLDAP systems), you -# probably want to use this value instead: "(memberUid={member_uid})" -# Note that this filter is &-combined with the group_filter_format query -# specifying a wildcard for the group name. So it will only find groups. -# member_group_filter_format: "(member={member_dn})" - # (optional) string_encoding (default value given below) # string_encoding specifies the Unicode string encoding used by the directory. # All values retrieved from the directory are converted to Unicode before being diff --git a/user_sync/connector/directory_ldap.py b/user_sync/connector/directory_ldap.py index b75ad7849..b8f807fb5 100755 --- a/user_sync/connector/directory_ldap.py +++ b/user_sync/connector/directory_ldap.py @@ -71,7 +71,6 @@ def __init__(self, caller_options): '(&(objectClass=user)(objectCategory=person)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))')) builder.set_string_value('group_member_filter_format', six.text_type( '(memberOf={group_dn})')) - builder.set_string_value('member_group_filter_format', None) builder.set_bool_value('require_tls_cert', False) builder.set_string_value('string_encoding', 'utf8') builder.set_string_value('user_identity_type_format', None) From 22bf233747c7b905c1fa7f3667114d69c189b476 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Tue, 21 Aug 2018 14:31:56 -0600 Subject: [PATCH 26/28] documentation improvements --- docs/en/user-manual/advanced_configuration.md | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/en/user-manual/advanced_configuration.md b/docs/en/user-manual/advanced_configuration.md index a257ab62d..ce956ddf1 100644 --- a/docs/en/user-manual/advanced_configuration.md +++ b/docs/en/user-manual/advanced_configuration.md @@ -683,26 +683,39 @@ Okta sync can use extended groups, attributes and after-mapping hooks. The name ## Additional Group Options -Some Adobe applications, such as Adobe Experience Manager, may have a -number of permission-based or role-based groups that need to be -represented in the Adobe Admin Console. These groups have no special -purpose in the console, apart from serving as containers for users. -However, because of how different Adobe products integrate with one -another, they must be represented in the console. +It is possible for the User Sync Tool to sync group relationships that +are not explicitly mapped out in `user-sync-config.yml`. Any LDAP group +that a user belongs to directly can be mapped and targeted to an Adobe +profile or user group using the `additional_groups` configuration +option. -The User Sync Tool can target these groups with the `additional_groups` -config option. +* Additional groups are identified with a regular expression +* The name of the mapped Adobe group can be customized with a regular +expression substitution string. + +Possible use cases: + +* Metadata such as department, employee type, etc +* ACL groups for [Adobe Experience Manager](https://www.adobe.com/marketing/experience-manager.html) +* Special-case group, role or profile assignment + +Note: This feature only works with the LDAP connector at this time. ### Additional Group Rules `additional_groups` is defined in `user-sync-config.yml` in the `groups` object. It specifies a list of rules to identify and filter groups present in the `memberOf` LDAP attribute, as well as rules that govern -how corresponding Adobe groups should be named. +how corresponding Adobe groups should be named. Groups that are +discovered with this feature will be added to a user's list of +targeted Adobe groups. + +### Additional Group Example -For example - suppose an Adobe Experience Manager customer would like +Suppose an Adobe Experience Manager customer would like to sync all AEM users to the admin console. They define a group -mapping in `user-sync-config.yml` like the following: +mapping in `user-sync-config.yml` to map the LDAP group `AEM-USERS` to +the Adobe group `Adobe Experience Manager`. ```yaml - directory_group: "AEM-USERS" @@ -710,16 +723,13 @@ mapping in `user-sync-config.yml` like the following: - "Adobe Experience Manager" ``` -All users that belong to the directory group `AEM-USERS` will be -assigned to the `Adobe Experience Manager` product profile. - This example company's AEM users fall into two broad categories - authors and publishers. These users already belong to LDAP groups that correspond to each role - `AEM-ACL-AUTHORS` and `AEM-ACL-PUBLISHERS`, respectively. Suppose this company wishes to assign users to these additional groups when syncing users. Assuming group membership information can be found in the `memberOf` user attribute, they can -leverage the `additional_group` config option: +leverage the `additional_groups` config option. ```yaml directory_users: @@ -728,7 +738,7 @@ directory_users: # ... group mappings, etc additional_groups: - source: "AEM-ACL-(.+)" - target: "AEM-(\1)" + target: "AEM-(\\1)" ``` `additional_groups` contains a list of additional group rules. `source` @@ -742,7 +752,7 @@ Adobe user group (`AEM-AUTHORS` or `AEM-PUBLISHERS`). Note: The company in this example can also add mappings for authors and publishers to the group mapping in `user-sync-config.yml`. The -key advantage to using the additional groups mechanism is that it +advantage to using the additional groups mechanism is that it will apply dynamically to any LDAP group that matches the regex `AEM-ACL-(.+)`. If additional AEM roles are introduced, they will be included in sync as long as they follow that naming convention - @@ -756,13 +766,16 @@ conjunction with the additional groups functionality detailed in the previous section, but it also applies to Adobe groups targeted in the group mapping as well as the extension config. -`group_sync_options` is defined in `user-sync-config.yml`. It contains -an object that currently has just one key - `auto_create`. -`auto_create` is boolean and is `False` by default. +`group_sync_options` is defined in the `directory_users` section in +`user-sync-config.yml`. It contains an object that currently has just +one key - `auto_create`. `auto_create` is boolean and is `False` by +default. To enable dynamic group creation, set `auto_create` to `True`: ```yaml +directory_users: + # ... additional directory config options group_sync_options: auto_create: True ``` @@ -776,8 +789,8 @@ following conditions are true: New groups are always created as user groups. The UMAPI does not support product profile creation, so the Sync Tool can't create them. If the Sync Tool is configured to target a misspelled profile name, or -a profile that doesn't exist, it will automatically create a group with -the specified name. +a profile that doesn't exist, it will automatically create a user group +with the specified name. --- From cb408345dce457127743380aaf016d9d253bd5e0 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Wed, 29 Aug 2018 11:15:49 -0600 Subject: [PATCH 27/28] not all users will have member_groups attribute --- user_sync/rules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/user_sync/rules.py b/user_sync/rules.py index 235dfc5c6..dd7ed9489 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -390,7 +390,8 @@ def read_desired_user_groups(self, mappings, directory_connector): self.logger.error('Target adobe group %s is not known; ignored', target_group_qualified_name) additional_groups = self.options.get('additional_groups', []) - for member_group in directory_user['member_groups']: + member_groups = directory_user.get('member_groups', []) + for member_group in member_groups: for group_rule in additional_groups: if group_rule['source'].match(member_group): rename_group = group_rule['source'].sub(group_rule['target'], member_group) @@ -398,7 +399,6 @@ def read_desired_user_groups(self, mappings, directory_connector): for umapi_name, umapi_info in six.iteritems(self.umapi_info_by_name): umapi_info.add_desired_group_for(user_key, rename_group) - self.logger.debug('Total directory users after filtering: %d', len(filtered_directory_user_by_user_key)) if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('Group work list: %s', dict([(umapi_name, umapi_info.get_desired_groups_by_user_key()) From 4aae87b240202134d2084be4a34b341ccfa00978 Mon Sep 17 00:00:00 2001 From: Andrew Dorton Date: Thu, 30 Aug 2018 15:35:39 -0600 Subject: [PATCH 28/28] don't auto-create groups if 'process_groups' is false --- docs/en/user-manual/advanced_configuration.md | 2 ++ examples/config files - basic/1 user-sync-config.yml | 2 ++ user_sync/rules.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/en/user-manual/advanced_configuration.md b/docs/en/user-manual/advanced_configuration.md index ce956ddf1..3ba877ef1 100644 --- a/docs/en/user-manual/advanced_configuration.md +++ b/docs/en/user-manual/advanced_configuration.md @@ -785,6 +785,8 @@ following conditions are true: 1. Group is targeted for at least one user 2. Group does not currently exist +3. The `--process-groups` command argument is set (or the equivalent + invocation option) New groups are always created as user groups. The UMAPI does not support product profile creation, so the Sync Tool can't create them. diff --git a/examples/config files - basic/1 user-sync-config.yml b/examples/config files - basic/1 user-sync-config.yml index d3dfd46c3..04ac00ff9 100644 --- a/examples/config files - basic/1 user-sync-config.yml +++ b/examples/config files - basic/1 user-sync-config.yml @@ -218,6 +218,8 @@ directory_users: # NOTE: auto_create applies to any targeted Adobe group that doesn't exist. # This includes groups targeted through group mapping, extension config, # and the additional_groups functionality. + # The --process-groups command argument or equivalent invocation setting must + # be enabled for groups to be auto-created # group_sync_options: # auto_create: False diff --git a/user_sync/rules.py b/user_sync/rules.py index dd7ed9489..0be484382 100644 --- a/user_sync/rules.py +++ b/user_sync/rules.py @@ -176,7 +176,8 @@ def run(self, directory_groups, directory_connector, umapi_connectors): umapi_stats = JobStats('Push to UMAPI' if self.push_umapi else 'Sync with UMAPI', divider="-") umapi_stats.log_start(logger) if directory_connector is not None: - self.create_umapi_groups(umapi_connectors) + if self.options.get('process_groups'): + self.create_umapi_groups(umapi_connectors) self.sync_umapi_users(umapi_connectors) if self.will_process_strays: self.process_strays(umapi_connectors)