diff --git a/.pylintrc b/.pylintrc index a0fbc2e148..e099e8f37b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -76,21 +76,22 @@ confidence= # --disable=W" disable= C0330, # Black disagrees with and explicitly violates this: https://github.com/python/black/issues/48 - too-many-locals, + abstract-method, # TODO: Fix abstract methods arguments-differ, - too-many-lines, + cyclic-import, # TODO: Resolve cyclic imports fixme, - too-many-arguments, invalid-name, - too-many-instance-attributes, - len-as-condition, # TODO: Enable this check once pylint 2.4.0 is released and consumed due to the fix in https://github.com/PyCQA/pylint/issues/2684 import-error, # Since we run Pylint before any of our builds in tox, this will always fail - protected-access, # TODO: Fix access - abstract-method, # TODO: Fix abstract methods - useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported. - cyclic-import, # TODO: Resolve cyclic imports + import-outside-toplevel, no-self-use, # TODO: Convert methods to functions where appropriate + protected-access, # TODO: Fix access + signature-differs, # TODO: fix kwargs + too-many-arguments, too-many-branches, # TODO: Simplify or ignore as appropriate + too-many-instance-attributes, + too-many-lines, + too-many-locals, + useless-object-inheritance, # TODO: Enable this check and fix code once Python 2 is no longer supported. [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs diff --git a/setup.py b/setup.py index 8bb6525e97..14ecc4d7af 100644 --- a/setup.py +++ b/setup.py @@ -60,16 +60,16 @@ def read_version(): extras["test"] = ( [ extras["all"], - "tox==3.13.1", + "tox==3.15.1", "flake8", - "pytest==4.4.1", + "pytest==4.6.10", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "mock", "contextlib2", "awslogs", - "black==19.3b0 ; python_version >= '3.6'", + "black==19.10b0 ; python_version >= '3.6'", "stopit==1.1.2", "apache-airflow==1.10.5", "fabric>=2.0", diff --git a/src/sagemaker/amazon/common.py b/src/sagemaker/amazon/common.py index 1346e7059e..2be29e7a33 100644 --- a/src/sagemaker/amazon/common.py +++ b/src/sagemaker/amazon/common.py @@ -261,11 +261,11 @@ def read_recordio(f): """ while True: try: - read_kmagic, = struct.unpack("I", f.read(4)) + (read_kmagic,) = struct.unpack("I", f.read(4)) except struct.error: return assert read_kmagic == _kmagic - len_record, = struct.unpack("I", f.read(4)) + (len_record,) = struct.unpack("I", f.read(4)) pad = (((len_record + 3) >> 2) << 2) - len_record yield f.read(len_record) if pad: diff --git a/src/sagemaker/cli/common.py b/src/sagemaker/cli/common.py index 660fed64f5..7256937bc7 100644 --- a/src/sagemaker/cli/common.py +++ b/src/sagemaker/cli/common.py @@ -41,7 +41,7 @@ def __init__(self, args): self.script = args.script self.instance_type = args.instance_type self.instance_count = args.instance_count - self.environment = {k: v for k, v in (kv.split("=") for kv in args.env)} + self.environment = dict((kv.split("=") for kv in args.env)) self.session = sagemaker.Session() diff --git a/src/sagemaker/rl/estimator.py b/src/sagemaker/rl/estimator.py index a065bafb07..60c67ef5de 100644 --- a/src/sagemaker/rl/estimator.py +++ b/src/sagemaker/rl/estimator.py @@ -362,10 +362,10 @@ def _validate_framework_format(cls, framework): Args: framework: """ - if framework and framework not in RLFramework: + if framework and framework not in list(RLFramework): raise ValueError( - "Invalid type: {}, valid RL frameworks types are: [{}]".format( - framework, [t for t in RLFramework] + "Invalid type: {}, valid RL frameworks types are: {}".format( + framework, list(RLFramework) ) ) @@ -375,11 +375,9 @@ def _validate_toolkit_format(cls, toolkit): Args: toolkit: """ - if toolkit and toolkit not in RLToolkit: + if toolkit and toolkit not in list(RLToolkit): raise ValueError( - "Invalid type: {}, valid RL toolkits types are: [{}]".format( - toolkit, [t for t in RLToolkit] - ) + "Invalid type: {}, valid RL toolkits types are: {}".format(toolkit, list(RLToolkit)) ) @classmethod diff --git a/src/sagemaker/tuner.py b/src/sagemaker/tuner.py index 164b06ae71..7a74a95e24 100644 --- a/src/sagemaker/tuner.py +++ b/src/sagemaker/tuner.py @@ -92,10 +92,10 @@ def __init__(self, warm_start_type, parents): warm start the new tuning job. """ - if warm_start_type not in WarmStartTypes: + if warm_start_type not in list(WarmStartTypes): raise ValueError( - "Invalid type: {}, valid warm start types are: [{}]".format( - warm_start_type, [t for t in WarmStartTypes] + "Invalid type: {}, valid warm start types are: {}".format( + warm_start_type, list(WarmStartTypes) ) ) diff --git a/src/sagemaker/workflow/airflow.py b/src/sagemaker/workflow/airflow.py index 54ff0ee5df..bf3be24d5d 100644 --- a/src/sagemaker/workflow/airflow.py +++ b/src/sagemaker/workflow/airflow.py @@ -315,15 +315,17 @@ def tuning_config(tuner, inputs, job_name=None, include_cls_metadata=False, mini } if tuner.estimator: - tune_config[ - "TrainingJobDefinition" - ], s3_operations = _extract_training_config_from_estimator( + ( + tune_config["TrainingJobDefinition"], + s3_operations, + ) = _extract_training_config_from_estimator( tuner, inputs, include_cls_metadata, mini_batch_size ) else: - tune_config[ - "TrainingJobDefinitions" - ], s3_operations = _extract_training_config_list_from_estimator_dict( + ( + tune_config["TrainingJobDefinitions"], + s3_operations, + ) = _extract_training_config_list_from_estimator_dict( tuner, inputs, include_cls_metadata, mini_batch_size ) diff --git a/tests/integ/test_monitoring_files.py b/tests/integ/test_monitoring_files.py index d91042c394..c4be7eb017 100644 --- a/tests/integ/test_monitoring_files.py +++ b/tests/integ/test_monitoring_files.py @@ -319,7 +319,7 @@ def test_constraint_violations_object_creation_from_file_path_with_customization def test_constraint_violations_object_creation_from_file_path_without_customizations( - sagemaker_session + sagemaker_session, ): constraint_violations = ConstraintViolations.from_file_path( constraint_violations_file_path=os.path.join( @@ -354,7 +354,7 @@ def test_constraint_violations_object_creation_from_string_with_customizations( def test_constraint_violations_object_creation_from_string_without_customizations( - sagemaker_session + sagemaker_session, ): with open(os.path.join(tests.integ.DATA_DIR, "monitor/constraint_violations.json"), "r") as f: file_body = f.read() @@ -404,7 +404,7 @@ def test_constraint_violations_object_creation_from_s3_uri_with_customizations( def test_constraint_violations_object_creation_from_s3_uri_without_customizations( - sagemaker_session + sagemaker_session, ): with open(os.path.join(tests.integ.DATA_DIR, "monitor/constraint_violations.json"), "r") as f: file_body = f.read() diff --git a/tests/integ/test_multi_variant_endpoint.py b/tests/integ/test_multi_variant_endpoint.py index eddcc11768..83c57c7776 100644 --- a/tests/integ/test_multi_variant_endpoint.py +++ b/tests/integ/test_multi_variant_endpoint.py @@ -53,8 +53,8 @@ tests.integ.DATA_DIR, "sparkml_model", "mleap_model.tar.gz" ) SPARK_ML_DEFAULT_VARIANT_NAME = ( - "AllTraffic" -) # default defined in src/sagemaker/session.py def production_variant + "AllTraffic" # default defined in src/sagemaker/session.py def production_variant +) SPARK_ML_WRONG_VARIANT_NAME = "WRONG_VARIANT" SPARK_ML_TEST_DATA = "1.0,C,38.0,71.5,1.0,female" SPARK_ML_MODEL_SCHEMA = json.dumps( diff --git a/tests/integ/test_tfs.py b/tests/integ/test_tfs.py index 00fb6fdb75..f27b76a4f0 100644 --- a/tests/integ/test_tfs.py +++ b/tests/integ/test_tfs.py @@ -163,7 +163,7 @@ def test_predict_with_entry_point(tfs_predictor_with_model_and_entry_point_same_ @pytest.mark.local_mode def test_predict_with_model_and_entry_point_and_dependencies_separated( - tfs_predictor_with_model_and_entry_point_and_dependencies + tfs_predictor_with_model_and_entry_point_and_dependencies, ): input_data = {"instances": [1.0, 2.0, 5.0]} expected_result = {"predictions": [4.0, 4.5, 6.0]} diff --git a/tests/unit/test_image.py b/tests/unit/test_image.py index 9c5ecdbdbd..cf17441fea 100644 --- a/tests/unit/test_image.py +++ b/tests/unit/test_image.py @@ -799,7 +799,7 @@ def test__aws_credentials_with_long_lived_credentials(): @patch("sagemaker.local.image._aws_credentials_available_in_metadata_service") def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_having_credentials( - mock + mock, ): credentials = Credentials( access_key=_random_string(), secret_key=_random_string(), token=_random_string() @@ -814,7 +814,7 @@ def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_ @patch("sagemaker.local.image._aws_credentials_available_in_metadata_service") def test__aws_credentials_with_short_lived_credentials_and_ec2_metadata_service_having_no_credentials( - mock + mock, ): credentials = Credentials( access_key=_random_string(), secret_key=_random_string(), token=_random_string() diff --git a/tests/unit/test_tuner.py b/tests/unit/test_tuner.py index 490ae96145..67de14bc50 100644 --- a/tests/unit/test_tuner.py +++ b/tests/unit/test_tuner.py @@ -517,7 +517,7 @@ def test_attach_tuning_job_with_estimator_from_hyperparameters(sagemaker_session def test_attach_tuning_job_with_estimator_from_hyperparameters_with_early_stopping( - sagemaker_session + sagemaker_session, ): job_details = copy.deepcopy(TUNING_JOB_DETAILS) job_details["HyperParameterTuningJobConfig"]["TrainingJobEarlyStoppingType"] = "Auto" diff --git a/tox.ini b/tox.ini index 78cd0f1d4a..a376b68e4e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36,py37 +envlist = black-format,flake8,pylint,twine,sphinx,doc8,py27,py36,py37,py38 skip_missing_interpreters = False @@ -80,7 +80,7 @@ basepython = python3 skipdist = true skip_install = true deps = - pylint==2.3.1 + pylint==2.5.2 commands = python -m pylint --rcfile=.pylintrc -j 0 src/sagemaker @@ -129,13 +129,13 @@ commands = doc8 [testenv:black-format] # Used during development (before committing) to format .py files. basepython = python3 -deps = black==19.3b0 +deps = black==19.10b0 commands = black -l 100 ./ [testenv:black-check] # Used by automated build steps to check that all files are properly formatted. basepython = python3 -deps = black==19.3b0 +deps = black==19.10b0 commands = black -l 100 --check ./