v1.1 - third try

.gitignore

@@ -9,7 +9,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
-build/
+/build/
 develop-eggs/
 dist/
 downloads/

RELEASE_NOTES.md

@@ -0,0 +1,34 @@
+# Release Notes for User Sync Tool Version 1.1 
+2017-03-03.
+
+## Updating from prior versions
+
+This version has 2 new required configuration items.  If you are using an earlier version, you will need to add the following entires to your main user-sync-config.yaml file:
+
+    limits:
+        max_deletions_per_run: 10    # if --remove-nonexistent-users is specified, this is the most users that will be removed.  Others will be left for a later run.  A critical message will be logged.
+        max_missing_users: 200       # if more than this number of user accounts are not found in the directory, user sync will abort with an error and a critical message will be logged.
+
+To update your installation, download the release for your platform and replace the user-sync or user-sync.pex file with the new one.  Same the old one in case of problems
+
+Because this version contains a more aggressive --process-groups function (a bug fix) you may want to run with --test-only first and make sure nothing unexpected is happening.
+
+## New Features
+
+1. Ability to specify additional directory attributes to load and specify a code snippet to implement complex mappings from directory information to Adobe user information and group membership.  This is covered in more detail in the updated documentastion.
+
+2. Releases for different platforms are packaged separately so you only have to download the platform(s) you want.  You do have to download the example configuration files and documentation separately.
+
+3. User removal limits and guards.  There are some new features to prevent user sync from accidentally removing large numbers of users in the event of misconfiguration or changes in the directory which result in users not being returned from queries.
+
+
+
+## Changes in Behavior
+
+A bug in --process-groups was fixed.  Previously, users present in the Adobe admin console but not in the directory were not removed from groups that were mapped in the user sync configuration file.  This is now fixed.  A group (user group or product configuration) that is mapped from a directory group is assumed to be under user sync control and any users in such groups that are not in the directory and in groups in the directory mapped to those Adobe groups are removed.
+
+This release of user sync should be compatible and have no other behavior changes.
+
+## Compatibility with Prior Versions
+
+Other than as noted above, existing configuration files and should work and have the same behavior.

examples/example.user-sync-config.yml

@@ -1,22 +1,22 @@
 dashboard:
   # specifies the configurations for the Adobe Enterprise Dashboards.  
   # By default, it would look for dashboard-owning-config.yml and
-  # dashboard-trustee-*-config.yml in the configuration path,
-  # with the yml's identifying the owning organization and trustee organizations
+  # dashboard-accessor-*-config.yml in the configuration path,
+  # with the yml's identifying the owning organization and accessor organizations
   # respectively.
   #  
   # You can also specify the configurations under this section too,
-  # with keys owning and trustees.
+  # with keys owning and accessors.
   #
   # Examples:
   # owning: example.dashboard-config.yml
-  # trustees:
+  # accessors:
   #   org1: example.dashboard-config.yml    
 
-  # specifies the filename format for the trustee org configurations.  
+  # specifies the filename format for the accessor org configurations.  
   # a filename that matches the format will have the organization name extracted 
   # from the filename. Default is:
-  # trustee_config_filename_format: "dashboard-trustee-{organization_name}-config.yml"
+  # accessor_config_filename_format: "dashboard-accessor-{organization_name}-config.yml"
 
 directory:
   # (optional) Default country code to use if directory doesn't provide one for a user [Must be two-letter ISO-3166 code - see https://en.wikipedia.org/wiki/ISO_3166-1]
@@ -45,7 +45,7 @@ directory:
     # dashboard_groups: a list of strings identifying the dashboard groups.
     #
     # a group in dashboard_groups can be qualified with, the first part being
-    # the trustee organization name.
+    # the accessor organization name.
     # e.g. org1::Default Acrobat Pro DC configuration
     #
     # examples:
@@ -66,6 +66,10 @@ directory:
   # Default is:
   # user_identity_type: enterpriseID
 
+limits:
+    max_deletions_per_run: 10    # if --remove-nonexistent-users is specified, this is the most users that will be removed.  Others will be left for a later run.  A critical message will be logged.
+    max_missing_users: 200       # if more than this number of user accounts are not found in the directory, user sync will abort with an error and a critical message will be logged.
+
 logging:
   # specifies whether you wish to generate a log file
   # 'True' or 'False'

tests/config_test.py

@@ -55,13 +55,13 @@ def test_get_directory_connector_options(self, mock_dict, mock_connector_conf):
     @mock.patch('user_sync.config.ConfigLoader.create_dashboard_options')
     @mock.patch('glob.glob1')
     @mock.patch('user_sync.config.ConfigLoader.parse_string')
-    def test_get_dashboard_options_for_trustees(self, mock_parse, mock_glob, mock_create_dash, mock_get_dict):
+    def test_get_dashboard_options_for_accessors(self, mock_parse, mock_glob, mock_create_dash, mock_get_dict):
         mock_create_dash.return_value = {'create_dash'}
         mock_glob.return_value = {''}
         mock_parse.return_value = {'organization_name': 'testOrgName'}
 
-        self.assertEquals(self.conf_load.get_dashboard_options_for_trustees(), {'testOrgName': set(['create_dash'])},
-                          'We return with trustee option in the expected format')
+        self.assertEquals(self.conf_load.get_dashboard_options_for_accessors(), {'testOrgName': set(['create_dash'])},
+                          'We return with accessor option in the expected format')
         self.assertEquals(mock_create_dash.call_count, 1, 'create dashboard options was called')
 
     def test_get_dict_from_sources_dict(self):
@@ -88,19 +88,26 @@ def test_create_dashboard_options(self, mock_dict):
 
     @mock.patch('user_sync.config.DictConfig.get_string')
     @mock.patch('user_sync.config.DictConfig.get_dict_config')
+    @mock.patch('user_sync.config.DictConfig.get_list_config')
     @mock.patch('user_sync.identity_type.parse_identity_type')
-    def test_get_rule_options(self, mock_id_type,mock_get_dict,mock_get_string):
+    def test_get_rule_options(self, mock_id_type,mock_get_dict,mock_get_list,mock_get_string):
         mock_id_type.return_value = 'new_acc'
         mock_get_dict.return_value = tests.helper.MockGetString()
+        mock_get_list.return_value = tests.helper.MockGetString()
         self.assertEquals(self.conf_load.get_rule_options(), {'username_filter_regex': None,
                                                               'update_user_info': True,
                                                               'manage_groups': True,
+                                                              'max_deletions_per_run': 1,
+                                                              'max_missing_users': 1,
                                                               'new_account_type': 'new_acc',
                                                               'directory_group_filter': None,
                                                               'default_country_code': 'test',
                                                               'remove_user_key_list': None,
                                                               'remove_list_output_path': None,
-                                                              'remove_nonexistent_users': False},
+                                                              'remove_nonexistent_users': False,
+                                                              'after_mapping_hook': None,
+                                                              'extended_attributes': None,
+                                                              },
                           'rule options are returned')
 
     def test_parse_string(self):

tests/helper.py

@@ -85,4 +85,10 @@ def create_action_manager():
 
 class MockGetString():
     def get_string(self,test1,test2):
-        return 'test'
+        return 'test'
+
+    def get_int(self,test1):
+        return 1
+
+    def iter_dict_configs(self):
+        return iter([])

tests/rules_test.py

@@ -30,14 +30,14 @@ class RulesTest(unittest.TestCase):
 
     def test_normal(self):
         owning_organization_name = user_sync.rules.OWNING_ORGANIZATION_NAME
-        trustee_1_organization_name = "trustee1"
+        accessor_1_organization_name = "accessor1"
         directory_group_1 = 'acrobat1' 
         directory_group_2 = 'acrobat2' 
         owning_group_11 = 'acrobat11' 
         owning_group_12 = 'acrobat12' 
         owning_group_21 = 'acrobat21' 
         directory_groups = {
-            directory_group_1: [user_sync.rules.Group(owning_group_11, owning_organization_name), user_sync.rules.Group('acrobat12', trustee_1_organization_name)],
+            directory_group_1: [user_sync.rules.Group(owning_group_11, owning_organization_name), user_sync.rules.Group('acrobat12', accessor_1_organization_name)],
             directory_group_2: [user_sync.rules.Group(owning_group_21, owning_organization_name)]
         }
         all_users = [tests.helper.create_test_user([directory_group_1]), 
@@ -53,19 +53,19 @@ def test_normal(self):
         owning_user_1['groups'] = [owning_group_11]
         owning_users.append(owning_user_1)
 
-        def mock_load_users_and_groups(groups):
+        def mock_load_users_and_groups(groups, extended_attributes=None):
             return (True, list(all_users))
         mock_directory_connector = mock.mock.create_autospec(user_sync.connector.directory.DirectoryConnector)
         mock_directory_connector.load_users_and_groups = mock_load_users_and_groups
 
         owning_commands_list = []    
         mock_owning_dashboard_connector = self.create_mock_dashboard_connector(owning_users, owning_commands_list)
 
-        trustee_commands_list = []    
-        mock_trustee_dashboard_connector = self.create_mock_dashboard_connector([], trustee_commands_list)
+        accessor_commands_list = []    
+        mock_accessor_dashboard_connector = self.create_mock_dashboard_connector([], accessor_commands_list)
 
         dashboard_connectors = user_sync.rules.DashboardConnectors(mock_owning_dashboard_connector, {
-            trustee_1_organization_name: mock_trustee_dashboard_connector
+            accessor_1_organization_name: mock_accessor_dashboard_connector
         })
 
         rule_processor = user_sync.rules.RuleProcessor({})
@@ -92,18 +92,18 @@ def mock_load_users_and_groups(groups):
         commands.add_user(self.create_user_attributes_for_commands(user, rule_options['update_user_info']))
         expected_owning_commands_list.append(commands)
 
-        expected_trustee_commands_list = []
+        expected_accessor_commands_list = []
         user = all_users[0]
         commands = tests.helper.create_dashboard_commands(user)
         commands.add_groups(set([owning_group_12]))
-        expected_trustee_commands_list.append(commands)
+        expected_accessor_commands_list.append(commands)
 
         tests.helper.assert_equal_dashboard_commands_list(self, expected_owning_commands_list, owning_commands_list)
-        tests.helper.assert_equal_dashboard_commands_list(self, expected_trustee_commands_list, trustee_commands_list)
+        tests.helper.assert_equal_dashboard_commands_list(self, expected_accessor_commands_list, accessor_commands_list)
 
     # default country code tests
-
-    def _do_country_code_test(self, mock_dashboard_commands, mock_connectors, identity_type, default_country_code, user_country_code, expected_country_code):
+    @mock.patch('logging.getLogger')
+    def _do_country_code_test(self, mock_dashboard_commands, mock_connectors, identity_type, default_country_code, user_country_code, expected_country_code, mock_logger):
         expected_result = {'lastname': 'User1', 'email': '[email protected]', 'firstname': '!Openldap CCE', 'option': 'updateIfAlreadyExists'}
         if (expected_country_code):
             expected_result['country'] = expected_country_code
@@ -121,7 +121,11 @@ def _do_country_code_test(self, mock_dashboard_commands, mock_connectors, identi
                                      'uid': '001'}
         }
         mock_rules.add_dashboard_user('[email protected]', mock_connectors)
-        mock_dashboard_commands.return_value.add_user.assert_called_with(expected_result)
+
+        if (identity_type == 'federatedID' and default_country_code == None and user_country_code == None):
+            mock_rules.logger.error.assert_called_with('User %s cannot be added as it has a blank country code and no default has been specified.','[email protected]')
+        else:
+            mock_dashboard_commands.return_value.add_user.assert_called_with(expected_result)
 
     # federatedId
     @mock.patch('user_sync.rules.DashboardConnectors')

user_sync/app.py

@@ -92,8 +92,8 @@ def init_log(logging_config):
     builder = user_sync.config.OptionsBuilder(logging_config)
     builder.set_bool_value('log_to_file', False)
     builder.set_string_value('file_log_directory', 'logs')
-    builder.set_string_value('file_log_level', 'debug')
-    builder.set_string_value('console_log_level', None)    
+    builder.set_string_value('file_log_level', 'info')
+    builder.set_string_value('console_log_level', 'info')
     options = builder.get_options()
 
     level_lookup = {
@@ -105,11 +105,18 @@ def init_log(logging_config):
     }
 
     console_log_level = level_lookup.get(options['console_log_level'])
-    if (console_log_level != None):
-        console_log_handler.setLevel(console_log_level)
-
+    if (console_log_level == None):
+        console_log_level = logging.INFO
+        logger.log(logging.WARNING, 'Unknown console log level: %s setting to info' % options['console_log_level'])
+    console_log_handler.setLevel(console_log_level)
+
+
     if options['log_to_file'] == True:
-        file_log_level = level_lookup.get(options['file_log_level'], logging.NOTSET)
+        unknown_file_log_level = False
+        file_log_level = level_lookup.get(options['file_log_level'])
+        if (file_log_level == None):
+            file_log_level = logging.INFO
+            unknown_file_log_level = True
         file_log_directory = options['file_log_directory']
         if not os.path.exists(file_log_directory):
             os.makedirs(file_log_directory)
@@ -119,6 +126,8 @@ def init_log(logging_config):
         fileHandler.setLevel(file_log_level)
         fileHandler.setFormatter(logging.Formatter(LOG_STRING_FORMAT, LOG_DATE_FORMAT))        
         logging.getLogger().addHandler(fileHandler)
+        if (unknown_file_log_level == True):
+            logger.log(logging.WARNING, 'Unknown file log level: %s setting to info' % options['file_log_level'])
 
 def begin_work(config_loader):
     '''
@@ -127,7 +136,7 @@ def begin_work(config_loader):
 
     directory_groups = config_loader.get_directory_groups()
     owning_dashboard_config = config_loader.get_dashboard_options_for_owning()
-    trustee_dashboard_configs = config_loader.get_dashboard_options_for_trustees()
+    accessor_dashboard_configs = config_loader.get_dashboard_options_for_accessors()
     rule_config = config_loader.get_rule_options()
 
     referenced_organization_names = set()
@@ -136,10 +145,10 @@ def begin_work(config_loader):
             organization_name = group.organization_name
             if (organization_name != user_sync.rules.OWNING_ORGANIZATION_NAME):
                 referenced_organization_names.add(organization_name)
-    referenced_organization_names.difference_update(trustee_dashboard_configs.iterkeys())
+    referenced_organization_names.difference_update(accessor_dashboard_configs.iterkeys())
 
     if (len(referenced_organization_names) > 0):
-        raise user_sync.error.AssertionException('dashboard_groups have references to unknown trustee dashboards: %s' % referenced_organization_names) 
+        raise user_sync.error.AssertionException('dashboard_groups have references to unknown accessor dashboards: %s' % referenced_organization_names) 
 
     directory_connector = None
     directory_connector_options = None
@@ -155,11 +164,11 @@ def begin_work(config_loader):
         directory_connector.initialize(directory_connector_options)
 
     dashboard_owning_connector = user_sync.connector.dashboard.DashboardConnector("owning", owning_dashboard_config)
-    dashboard_trustee_connectors = {}    
-    for trustee_organization_name, trustee_config in trustee_dashboard_configs.iteritems():
-        dashboard_trustee_conector = user_sync.connector.dashboard.DashboardConnector("trustee.%s" % trustee_organization_name, trustee_config)
-        dashboard_trustee_connectors[trustee_organization_name] = dashboard_trustee_conector 
-    dashboard_connectors = user_sync.rules.DashboardConnectors(dashboard_owning_connector, dashboard_trustee_connectors)
+    dashboard_accessor_connectors = {}    
+    for accessor_organization_name, accessor_config in accessor_dashboard_configs.iteritems():
+        dashboard_accessor_conector = user_sync.connector.dashboard.DashboardConnector("accessor.%s" % accessor_organization_name, accessor_config)
+        dashboard_accessor_connectors[accessor_organization_name] = dashboard_accessor_conector 
+    dashboard_connectors = user_sync.rules.DashboardConnectors(dashboard_owning_connector, dashboard_accessor_connectors)
 
     rule_processor = user_sync.rules.RuleProcessor(rule_config)
     if (len(directory_groups) == 0 and rule_processor.will_manage_groups()):
@@ -266,7 +275,7 @@ def main():
 
     except user_sync.error.AssertionException as e:
         if (not e.is_reported()):
-            logger.error(e.message)
+            logger.critical(e.message)
             e.set_reported()    
     except:
         try: