diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b5c5e7..985bae8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: ruff name: ruff lint - args: ["--select", "I", "--fix"] + args: ["--fix"] files: ^pysqa/ - id: ruff-format name: ruff format diff --git a/pyproject.toml b/pyproject.toml index 37918df..d14694b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,43 @@ include = ["pysqa*"] [tool.setuptools.dynamic] version = {attr = "pysqa.__version__"} +[tool.ruff] +exclude = [".ci_support", "docs", "notebooks", "tests", "setup.py", "_version.py"] + +[tool.ruff.lint] +select = [ + # pycodestyle + "E", + # Pyflakes + "F", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # flake8-simplify + "SIM", + # isort + "I", + # flake8-comprehensions + "C4", + # eradicate + "ERA", + # pylint + "PL", +] +ignore = [ + # ignore exception naming + "B904", + # ignore line-length violations + "E501", + # Too many arguments in function definition + "PLR0913", + # Too many branches + "PLR0912", + # Too many statements + "PLR0915", +] + [tool.versioneer] VCS = "git" style = "pep440-pre" diff --git a/pysqa/__init__.py b/pysqa/__init__.py index b01eb50..3324c26 100644 --- a/pysqa/__init__.py +++ b/pysqa/__init__.py @@ -2,5 +2,5 @@ from ._version import get_versions -__all__ = [QueueAdapter] +__all__ = ["QueueAdapter"] __version__ = get_versions()["version"] diff --git a/pysqa/base/abstract.py b/pysqa/base/abstract.py index efd0c5c..ac2d3cf 100644 --- a/pysqa/base/abstract.py +++ b/pysqa/base/abstract.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Optional, Union +from typing import Optional, Union import pandas from jinja2 import Template @@ -15,7 +15,7 @@ def submit_job( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[str]] = None, + dependency_list: Optional[list[str]] = None, command: Optional[str] = None, submission_template: Optional[Union[str, Template]] = None, **kwargs, @@ -67,7 +67,7 @@ def get_status_of_job(self, process_id: int) -> Union[str, None]: pass @abstractmethod - def get_status_of_jobs(self, process_id_lst: List[int]) -> List[str]: + def get_status_of_jobs(self, process_id_lst: list[int]) -> list[str]: """ Get the status of multiple jobs. diff --git a/pysqa/base/cmd.py b/pysqa/base/cmd.py index 0661a41..a24aa02 100644 --- a/pysqa/base/cmd.py +++ b/pysqa/base/cmd.py @@ -123,7 +123,7 @@ def command_line( elif mode_list: working_directory = os.path.abspath(os.path.expanduser(working_directory)) remote_dirs, remote_files = [], [] - for p, folder, files in os.walk(working_directory): + for p, _folder, files in os.walk(working_directory): remote_dirs.append(p) remote_files += [os.path.join(p, f) for f in files] print( diff --git a/pysqa/base/config.py b/pysqa/base/config.py index 6b2d750..cb8d8f1 100644 --- a/pysqa/base/config.py +++ b/pysqa/base/config.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional, Tuple, Union +from typing import Optional, Union import pandas import yaml @@ -10,13 +10,13 @@ from pysqa.base.validate import check_queue_parameters, value_error_if_none -class Queues(object): +class Queues: """ Queues is an abstract class simply to make the list of queues available for auto completion. This is mainly used in interactive environments like jupyter. """ - def __init__(self, list_of_queues: List[str]): + def __init__(self, list_of_queues: list[str]): """ Initialize the Queues object. @@ -45,7 +45,7 @@ def __getattr__(self, item: str) -> str: else: raise AttributeError - def __dir__(self) -> List[str]: + def __dir__(self) -> list[str]: """ Get the list of queues. @@ -189,7 +189,7 @@ def check_queue_parameters( run_time_max: Optional[int] = None, memory_max: Optional[int] = None, active_queue: Optional[dict] = None, - ) -> Tuple[ + ) -> tuple[ Union[float, int, None], Union[float, int, None], Union[float, int, None] ]: """ @@ -223,7 +223,7 @@ def _job_submission_template( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, command: Optional[str] = None, **kwargs, ) -> str: @@ -297,8 +297,8 @@ def _load_templates(queue_lst_dict: dict, directory: str = ".") -> None: directory (str, optional): The directory where the queue template files are located. Defaults to ".". """ for queue_dict in queue_lst_dict.values(): - if "script" in queue_dict.keys(): - with open(os.path.join(directory, queue_dict["script"]), "r") as f: + if "script" in queue_dict: + with open(os.path.join(directory, queue_dict["script"])) as f: try: queue_dict["template"] = Template(f.read()) except TemplateSyntaxError as error: @@ -321,5 +321,5 @@ def read_config(file_name: str = "queue.yaml") -> dict: Returns: dict: The parsed configuration as a dictionary. """ - with open(file_name, "r") as f: + with open(file_name) as f: return yaml.load(f, Loader=yaml.FullLoader) diff --git a/pysqa/base/core.py b/pysqa/base/core.py index 496b293..5838951 100644 --- a/pysqa/base/core.py +++ b/pysqa/base/core.py @@ -2,7 +2,7 @@ import importlib import os import subprocess -from typing import List, Optional, Tuple, Union +from typing import Optional, Union import pandas from jinja2 import Template @@ -52,7 +52,7 @@ def execute_command( split_output: bool = True, shell: bool = False, error_filename: str = "pysqa.err", -) -> Union[str, List[str]]: +) -> Union[str, list[str]]: """ A wrapper around the subprocess.check_output function. @@ -96,7 +96,7 @@ def get_queue_commands(queue_type: str) -> Union[SchedulerCommands, None]: Returns: SchedulerCommands: queuing system commands class instance """ - if queue_type in queue_type_dict.keys(): + if queue_type in queue_type_dict: class_name = queue_type_dict[queue_type]["class_name"] module_name = queue_type_dict[queue_type]["module_name"] if module_name is not None and class_name is not None: @@ -129,10 +129,9 @@ def __init__( ): self._commands = get_queue_commands(queue_type=queue_type) if queue_type_dict[queue_type]["module_name"] is not None: - self._submission_template = getattr( - importlib.import_module(queue_type_dict[queue_type]["module_name"]), - "template", - ) + self._submission_template = importlib.import_module( + queue_type_dict[queue_type]["module_name"] + ).template self._execute_command_function = execute_command def submit_job( @@ -143,7 +142,7 @@ def submit_job( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[str]] = None, + dependency_list: Optional[list[str]] = None, command: Optional[str] = None, submission_template: Optional[Union[str, Template]] = None, **kwargs, @@ -277,7 +276,7 @@ def get_status_of_job(self, process_id: int) -> Union[str, None]: else: return None - def get_status_of_jobs(self, process_id_lst: List[int]) -> List[str]: + def get_status_of_jobs(self, process_id_lst: list[int]) -> list[str]: """ Get the status of multiple jobs. @@ -311,7 +310,7 @@ def _list_command_to_be_executed(self, queue_script_path: str) -> list: def _execute_command( self, - commands: Union[str, List[str]], + commands: Union[str, list[str]], working_directory: Optional[str] = None, split_output: bool = True, shell: bool = False, @@ -347,10 +346,10 @@ def _write_queue_script( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, command: Optional[str] = None, **kwargs, - ) -> Tuple[str, str]: + ) -> tuple[str, str]: """ Write the queue script to a file. @@ -399,7 +398,7 @@ def _job_submission_template( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, command: Optional[str] = None, **kwargs, ) -> str: diff --git a/pysqa/base/modular.py b/pysqa/base/modular.py index 382cf92..3fd210f 100644 --- a/pysqa/base/modular.py +++ b/pysqa/base/modular.py @@ -29,7 +29,7 @@ def __init__( directory: str = "~/.queues", execute_command: callable = execute_command, ): - super(ModularQueueAdapter, self).__init__( + super().__init__( config=config, directory=directory, execute_command=execute_command ) self._queue_to_cluster_dict = { @@ -212,4 +212,4 @@ def _switch_cluster_command(cluster_module: str): list: The switch cluster command. """ - return ["module", "--quiet", "swap", "cluster/{};".format(cluster_module)] + return ["module", "--quiet", "swap", f"cluster/{cluster_module};"] diff --git a/pysqa/base/remote.py b/pysqa/base/remote.py index 7d60a35..9b7319b 100644 --- a/pysqa/base/remote.py +++ b/pysqa/base/remote.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import List, Optional, Union +from typing import Optional, Union import pandas import paramiko @@ -85,7 +85,7 @@ def __init__( directory: str = "~/.queues", execute_command: callable = execute_command, ): - super(RemoteQueueAdapter, self).__init__( + super().__init__( config=config, directory=directory, execute_command=execute_command ) self._ssh_host = config["ssh_host"] @@ -93,56 +93,38 @@ def __init__( self._ssh_known_hosts = os.path.abspath( os.path.expanduser(config["known_hosts"]) ) - if "ssh_key" in config.keys(): + if "ssh_key" in config: self._ssh_key = os.path.abspath(os.path.expanduser(config["ssh_key"])) self._ssh_ask_for_password = False else: self._ssh_key = None - if "ssh_password" in config.keys(): + if "ssh_password" in config: self._ssh_password = config["ssh_password"] self._ssh_ask_for_password = False else: self._ssh_password = None - if "ssh_ask_for_password" in config.keys(): - self._ssh_ask_for_password = config["ssh_ask_for_password"] - else: - self._ssh_ask_for_password = False - if "ssh_key_passphrase" in config.keys(): - self._ssh_key_passphrase = config["ssh_key_passphrase"] - else: - self._ssh_key_passphrase = None - if "ssh_two_factor_authentication" in config.keys(): + self._ssh_ask_for_password = config.get("ssh_ask_for_password", False) + self._ssh_key_passphrase = config.get("ssh_key_passphrase") + if "ssh_two_factor_authentication" in config: self._ssh_two_factor_authentication = config[ "ssh_two_factor_authentication" ] else: self._ssh_two_factor_authentication = False - if "ssh_authenticator_service" in config.keys(): + if "ssh_authenticator_service" in config: self._ssh_authenticator_service = config["ssh_authenticator_service"] self._ssh_two_factor_authentication = True else: self._ssh_authenticator_service = None - if "ssh_proxy_host" in config.keys(): - self._ssh_proxy_host = config["ssh_proxy_host"] - else: - self._ssh_proxy_host = None + self._ssh_proxy_host = config.get("ssh_proxy_host") self._ssh_remote_config_dir = config["ssh_remote_config_dir"] self._ssh_remote_path = config["ssh_remote_path"] self._ssh_local_path = os.path.abspath( os.path.expanduser(config["ssh_local_path"]) ) - if "ssh_delete_file_on_remote" in config.keys(): - self._ssh_delete_file_on_remote = config["ssh_delete_file_on_remote"] - else: - self._ssh_delete_file_on_remote = True - if "ssh_port" in config.keys(): - self._ssh_port = config["ssh_port"] - else: - self._ssh_port = 22 - if "ssh_continous_connection" in config.keys(): - self._ssh_continous_connection = config["ssh_continous_connection"] - else: - self._ssh_continous_connection = False + self._ssh_delete_file_on_remote = config.get("ssh_delete_file_on_remote", True) + self._ssh_port = config.get("ssh_port", 22) + self._ssh_continous_connection = config.get("ssh_continous_connection", False) self._ssh_connection = None self._ssh_proxy_connection = None self._remote_flag = True @@ -571,7 +553,7 @@ def _execute_remote_command(self, command: str) -> str: else: ssh = self._open_ssh_connection() stdin, stdout, stderr = ssh.exec_command(command) - warnings.warn(stderr.read().decode()) + warnings.warn(message=stderr.read().decode(), stacklevel=2) output = stdout.read().decode() if not self._ssh_continous_connection: ssh.close() @@ -592,7 +574,7 @@ def _get_remote_working_dir(self, working_directory: str) -> str: os.path.relpath(working_directory, self._ssh_local_path), ) - def _create_remote_dir(self, directory: Union[str, List[str]]) -> None: + def _create_remote_dir(self, directory: Union[str, list[str]]) -> None: """ Creates a remote directory on the SSH server. @@ -628,7 +610,7 @@ def _transfer_data_to_remote(self, working_directory: str) -> None: ) file_dict = {} new_dir_list = [] - for p, folder, files in os.walk(working_directory): + for p, _folder, files in os.walk(working_directory): new_dir_list.append( self._get_file_transfer( file=p, diff --git a/pysqa/base/validate.py b/pysqa/base/validate.py index 77b7f43..cc52c8a 100644 --- a/pysqa/base/validate.py +++ b/pysqa/base/validate.py @@ -1,5 +1,5 @@ import re -from typing import Optional, Tuple, Union +from typing import Optional, Union def check_queue_parameters( @@ -7,7 +7,7 @@ def check_queue_parameters( cores: int = 1, run_time_max: Optional[int] = None, memory_max: Optional[int] = None, -) -> Tuple[Union[float, int, None], Union[float, int, None], Union[float, int, None]]: +) -> tuple[Union[float, int, None], Union[float, int, None], Union[float, int, None]]: """ Check the parameters of a queue. diff --git a/pysqa/queueadapter.py b/pysqa/queueadapter.py index 76bdb35..a18069e 100644 --- a/pysqa/queueadapter.py +++ b/pysqa/queueadapter.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional, Tuple, Union +from typing import Optional, Union import pandas from jinja2 import Template @@ -90,7 +90,7 @@ def __init__( else: raise ValueError() - def list_clusters(self) -> List[str]: + def list_clusters(self) -> list[str]: """ List available computing clusters for remote submission @@ -148,7 +148,7 @@ def remote_flag(self) -> bool: return False @property - def queue_list(self) -> Union[List[str], None]: + def queue_list(self) -> Union[list[str], None]: """ Get the list of available queues. @@ -194,7 +194,7 @@ def submit_job( cores: Optional[int] = None, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[str]] = None, + dependency_list: Optional[list[str]] = None, command: Optional[str] = None, submission_template: Optional[Union[str, Template]] = None, **kwargs, @@ -338,7 +338,7 @@ def get_status_of_job(self, process_id: int) -> str: """ return self._adapter.get_status_of_job(process_id=process_id) - def get_status_of_jobs(self, process_id_lst: List[int]) -> List[str]: + def get_status_of_jobs(self, process_id_lst: list[int]) -> list[str]: """ Get the status of multiple jobs. @@ -357,7 +357,7 @@ def check_queue_parameters( run_time_max: Optional[int] = None, memory_max: Optional[int] = None, active_queue: Optional[dict] = None, - ) -> Tuple[ + ) -> tuple[ Union[float, int, None], Union[float, int, None], Union[float, int, None] ]: """ diff --git a/pysqa/wrapper/abstract.py b/pysqa/wrapper/abstract.py index be127eb..2528d0c 100644 --- a/pysqa/wrapper/abstract.py +++ b/pysqa/wrapper/abstract.py @@ -1,6 +1,6 @@ import os from abc import ABC, abstractmethod -from typing import List, Optional, Union +from typing import Optional, Union import pandas from jinja2 import Template @@ -59,7 +59,7 @@ def render_submission_template( cores: int = 1, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, **kwargs, ) -> str: """ diff --git a/pysqa/wrapper/flux.py b/pysqa/wrapper/flux.py index c22921a..701ad0b 100644 --- a/pysqa/wrapper/flux.py +++ b/pysqa/wrapper/flux.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional, Union +from typing import Optional, Union import pandas from flux.job import JobID @@ -82,7 +82,7 @@ def render_submission_template( cores: int = 1, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, submission_template: Union[str, Template] = template, **kwargs, ) -> str: diff --git a/pysqa/wrapper/lsf.py b/pysqa/wrapper/lsf.py index 92b94c3..a1bcd14 100644 --- a/pysqa/wrapper/lsf.py +++ b/pysqa/wrapper/lsf.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional, Union +from typing import Optional, Union import pandas from jinja2 import Template @@ -79,7 +79,7 @@ def render_submission_template( cores: int = 1, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, submission_template: Union[str, Template] = template, **kwargs, ) -> str: diff --git a/pysqa/wrapper/moab.py b/pysqa/wrapper/moab.py index 5760a4c..2c1fcd3 100644 --- a/pysqa/wrapper/moab.py +++ b/pysqa/wrapper/moab.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional, Union +from typing import Optional, Union from jinja2 import Template @@ -58,7 +58,7 @@ def render_submission_template( cores: int = 1, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, submission_template: Union[str, Template] = template, **kwargs, ) -> str: diff --git a/pysqa/wrapper/sge.py b/pysqa/wrapper/sge.py index c4657e0..93e4261 100644 --- a/pysqa/wrapper/sge.py +++ b/pysqa/wrapper/sge.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional, Union +from typing import Optional, Union import defusedxml.ElementTree as ETree import pandas @@ -95,7 +95,7 @@ def render_submission_template( cores: int = 1, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, submission_template: Union[str, Template] = template, **kwargs, ) -> str: diff --git a/pysqa/wrapper/slurm.py b/pysqa/wrapper/slurm.py index aaa1a2d..a5fd631 100644 --- a/pysqa/wrapper/slurm.py +++ b/pysqa/wrapper/slurm.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional, Union +from typing import Optional, Union import pandas from jinja2 import Template @@ -97,7 +97,7 @@ def render_submission_template( cores: int = 1, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, submission_template: Union[str, Template] = template, **kwargs, ) -> str: diff --git a/pysqa/wrapper/torque.py b/pysqa/wrapper/torque.py index ed6a41f..05d436f 100644 --- a/pysqa/wrapper/torque.py +++ b/pysqa/wrapper/torque.py @@ -1,6 +1,6 @@ import os import re -from typing import List, Optional, Union +from typing import Optional, Union import pandas from jinja2 import Template @@ -118,7 +118,7 @@ def render_submission_template( cores: int = 1, memory_max: Optional[int] = None, run_time_max: Optional[int] = None, - dependency_list: Optional[List[int]] = None, + dependency_list: Optional[list[int]] = None, submission_template: Union[str, Template] = template, **kwargs, ) -> str: