Fixes using ruff (#383)

* Ruff fixes

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: jan-janssen

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
       - id: ruff
         name: ruff lint
-        args: ["--select", "I", "--fix"]
+        args: ["--fix"]
         files: ^pysqa/
       - id: ruff-format
         name: ruff format

pyproject.toml

@@ -53,6 +53,43 @@ include = ["pysqa*"]
 [tool.setuptools.dynamic]
 version = {attr = "pysqa.__version__"}
 
+[tool.ruff]
+exclude = [".ci_support", "docs", "notebooks", "tests", "setup.py", "_version.py"]
+
+[tool.ruff.lint]
+select = [
+    # pycodestyle
+    "E",
+    # Pyflakes
+    "F",
+    # pyupgrade
+    "UP",
+    # flake8-bugbear
+    "B",
+    # flake8-simplify
+    "SIM",
+    # isort
+    "I",
+    # flake8-comprehensions
+    "C4",
+    # eradicate
+    "ERA",
+    # pylint
+    "PL",
+]
+ignore = [
+    # ignore exception naming
+    "B904",
+    # ignore line-length violations
+    "E501",
+    # Too many arguments in function definition
+    "PLR0913",
+    # Too many branches
+    "PLR0912",
+    # Too many statements
+    "PLR0915",
+]
+
 [tool.versioneer]
 VCS = "git"
 style = "pep440-pre"

pysqa/__init__.py

@@ -2,5 +2,5 @@
 
 from ._version import get_versions
 
-__all__ = [QueueAdapter]
+__all__ = ["QueueAdapter"]
 __version__ = get_versions()["version"]

pysqa/base/abstract.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import pandas
 from jinja2 import Template
@@ -15,7 +15,7 @@ def submit_job(
         cores: Optional[int] = None,
         memory_max: Optional[int] = None,
         run_time_max: Optional[int] = None,
-        dependency_list: Optional[List[str]] = None,
+        dependency_list: Optional[list[str]] = None,
         command: Optional[str] = None,
         submission_template: Optional[Union[str, Template]] = None,
         **kwargs,
@@ -67,7 +67,7 @@ def get_status_of_job(self, process_id: int) -> Union[str, None]:
         pass
 
     @abstractmethod
-    def get_status_of_jobs(self, process_id_lst: List[int]) -> List[str]:
+    def get_status_of_jobs(self, process_id_lst: list[int]) -> list[str]:
         """
         Get the status of multiple jobs.
 

pysqa/base/cmd.py

@@ -123,7 +123,7 @@ def command_line(
         elif mode_list:
             working_directory = os.path.abspath(os.path.expanduser(working_directory))
             remote_dirs, remote_files = [], []
-            for p, folder, files in os.walk(working_directory):
+            for p, _folder, files in os.walk(working_directory):
                 remote_dirs.append(p)
                 remote_files += [os.path.join(p, f) for f in files]
             print(

pysqa/base/config.py

@@ -1,5 +1,5 @@
 import os
-from typing import List, Optional, Tuple, Union
+from typing import Optional, Union
 
 import pandas
 import yaml
@@ -10,13 +10,13 @@
 from pysqa.base.validate import check_queue_parameters, value_error_if_none
 
 
-class Queues(object):
+class Queues:
     """
     Queues is an abstract class simply to make the list of queues available for auto completion. This is mainly used in
     interactive environments like jupyter.
     """
 
-    def __init__(self, list_of_queues: List[str]):
+    def __init__(self, list_of_queues: list[str]):
         """
         Initialize the Queues object.
 
@@ -45,7 +45,7 @@ def __getattr__(self, item: str) -> str:
         else:
             raise AttributeError
 
-    def __dir__(self) -> List[str]:
+    def __dir__(self) -> list[str]:
         """
         Get the list of queues.
 
@@ -189,7 +189,7 @@ def check_queue_parameters(
         run_time_max: Optional[int] = None,
         memory_max: Optional[int] = None,
         active_queue: Optional[dict] = None,
-    ) -> Tuple[
+    ) -> tuple[
         Union[float, int, None], Union[float, int, None], Union[float, int, None]
     ]:
         """
@@ -223,7 +223,7 @@ def _job_submission_template(
         cores: Optional[int] = None,
         memory_max: Optional[int] = None,
         run_time_max: Optional[int] = None,
-        dependency_list: Optional[List[int]] = None,
+        dependency_list: Optional[list[int]] = None,
         command: Optional[str] = None,
         **kwargs,
     ) -> str:
@@ -297,8 +297,8 @@ def _load_templates(queue_lst_dict: dict, directory: str = ".") -> None:
             directory (str, optional): The directory where the queue template files are located. Defaults to ".".
         """
         for queue_dict in queue_lst_dict.values():
-            if "script" in queue_dict.keys():
-                with open(os.path.join(directory, queue_dict["script"]), "r") as f:
+            if "script" in queue_dict:
+                with open(os.path.join(directory, queue_dict["script"])) as f:
                     try:
                         queue_dict["template"] = Template(f.read())
                     except TemplateSyntaxError as error:
@@ -321,5 +321,5 @@ def read_config(file_name: str = "queue.yaml") -> dict:
     Returns:
         dict: The parsed configuration as a dictionary.
     """
-    with open(file_name, "r") as f:
+    with open(file_name) as f:
         return yaml.load(f, Loader=yaml.FullLoader)

pysqa/base/core.py

@@ -2,7 +2,7 @@
 import importlib
 import os
 import subprocess
-from typing import List, Optional, Tuple, Union
+from typing import Optional, Union
 
 import pandas
 from jinja2 import Template
@@ -52,7 +52,7 @@ def execute_command(
     split_output: bool = True,
     shell: bool = False,
     error_filename: str = "pysqa.err",
-) -> Union[str, List[str]]:
+) -> Union[str, list[str]]:
     """
     A wrapper around the subprocess.check_output function.
 
@@ -96,7 +96,7 @@ def get_queue_commands(queue_type: str) -> Union[SchedulerCommands, None]:
     Returns:
         SchedulerCommands: queuing system commands class instance
     """
-    if queue_type in queue_type_dict.keys():
+    if queue_type in queue_type_dict:
         class_name = queue_type_dict[queue_type]["class_name"]
         module_name = queue_type_dict[queue_type]["module_name"]
         if module_name is not None and class_name is not None:
@@ -129,10 +129,9 @@ def __init__(
     ):
         self._commands = get_queue_commands(queue_type=queue_type)
         if queue_type_dict[queue_type]["module_name"] is not None:
-            self._submission_template = getattr(
-                importlib.import_module(queue_type_dict[queue_type]["module_name"]),
-                "template",
-            )
+            self._submission_template = importlib.import_module(
+                queue_type_dict[queue_type]["module_name"]
+            ).template
         self._execute_command_function = execute_command
 
     def submit_job(
@@ -143,7 +142,7 @@ def submit_job(
         cores: Optional[int] = None,
         memory_max: Optional[int] = None,
         run_time_max: Optional[int] = None,
-        dependency_list: Optional[List[str]] = None,
+        dependency_list: Optional[list[str]] = None,
         command: Optional[str] = None,
         submission_template: Optional[Union[str, Template]] = None,
         **kwargs,
@@ -277,7 +276,7 @@ def get_status_of_job(self, process_id: int) -> Union[str, None]:
         else:
             return None
 
-    def get_status_of_jobs(self, process_id_lst: List[int]) -> List[str]:
+    def get_status_of_jobs(self, process_id_lst: list[int]) -> list[str]:
         """
         Get the status of multiple jobs.
 
@@ -311,7 +310,7 @@ def _list_command_to_be_executed(self, queue_script_path: str) -> list:
 
     def _execute_command(
         self,
-        commands: Union[str, List[str]],
+        commands: Union[str, list[str]],
         working_directory: Optional[str] = None,
         split_output: bool = True,
         shell: bool = False,
@@ -347,10 +346,10 @@ def _write_queue_script(
         cores: Optional[int] = None,
         memory_max: Optional[int] = None,
         run_time_max: Optional[int] = None,
-        dependency_list: Optional[List[int]] = None,
+        dependency_list: Optional[list[int]] = None,
         command: Optional[str] = None,
         **kwargs,
-    ) -> Tuple[str, str]:
+    ) -> tuple[str, str]:
         """
         Write the queue script to a file.
 
@@ -399,7 +398,7 @@ def _job_submission_template(
         cores: Optional[int] = None,
         memory_max: Optional[int] = None,
         run_time_max: Optional[int] = None,
-        dependency_list: Optional[List[int]] = None,
+        dependency_list: Optional[list[int]] = None,
         command: Optional[str] = None,
         **kwargs,
     ) -> str:

pysqa/base/modular.py

@@ -29,7 +29,7 @@ def __init__(
         directory: str = "~/.queues",
         execute_command: callable = execute_command,
     ):
-        super(ModularQueueAdapter, self).__init__(
+        super().__init__(
             config=config, directory=directory, execute_command=execute_command
         )
         self._queue_to_cluster_dict = {
@@ -212,4 +212,4 @@ def _switch_cluster_command(cluster_module: str):
             list: The switch cluster command.
 
         """
-        return ["module", "--quiet", "swap", "cluster/{};".format(cluster_module)]
+        return ["module", "--quiet", "swap", f"cluster/{cluster_module};"]

pysqa/base/remote.py

@@ -2,7 +2,7 @@
 import json
 import os
 import warnings
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import pandas
 import paramiko
@@ -85,64 +85,46 @@ def __init__(
         directory: str = "~/.queues",
         execute_command: callable = execute_command,
     ):
-        super(RemoteQueueAdapter, self).__init__(
+        super().__init__(
             config=config, directory=directory, execute_command=execute_command
         )
         self._ssh_host = config["ssh_host"]
         self._ssh_username = config["ssh_username"]
         self._ssh_known_hosts = os.path.abspath(
             os.path.expanduser(config["known_hosts"])
         )
-        if "ssh_key" in config.keys():
+        if "ssh_key" in config:
             self._ssh_key = os.path.abspath(os.path.expanduser(config["ssh_key"]))
             self._ssh_ask_for_password = False
         else:
             self._ssh_key = None
-        if "ssh_password" in config.keys():
+        if "ssh_password" in config:
             self._ssh_password = config["ssh_password"]
             self._ssh_ask_for_password = False
         else:
             self._ssh_password = None
-        if "ssh_ask_for_password" in config.keys():
-            self._ssh_ask_for_password = config["ssh_ask_for_password"]
-        else:
-            self._ssh_ask_for_password = False
-        if "ssh_key_passphrase" in config.keys():
-            self._ssh_key_passphrase = config["ssh_key_passphrase"]
-        else:
-            self._ssh_key_passphrase = None
-        if "ssh_two_factor_authentication" in config.keys():
+        self._ssh_ask_for_password = config.get("ssh_ask_for_password", False)
+        self._ssh_key_passphrase = config.get("ssh_key_passphrase")
+        if "ssh_two_factor_authentication" in config:
             self._ssh_two_factor_authentication = config[
                 "ssh_two_factor_authentication"
             ]
         else:
             self._ssh_two_factor_authentication = False
-        if "ssh_authenticator_service" in config.keys():
+        if "ssh_authenticator_service" in config:
             self._ssh_authenticator_service = config["ssh_authenticator_service"]
             self._ssh_two_factor_authentication = True
         else:
             self._ssh_authenticator_service = None
-        if "ssh_proxy_host" in config.keys():
-            self._ssh_proxy_host = config["ssh_proxy_host"]
-        else:
-            self._ssh_proxy_host = None
+        self._ssh_proxy_host = config.get("ssh_proxy_host")
         self._ssh_remote_config_dir = config["ssh_remote_config_dir"]
         self._ssh_remote_path = config["ssh_remote_path"]
         self._ssh_local_path = os.path.abspath(
             os.path.expanduser(config["ssh_local_path"])
         )
-        if "ssh_delete_file_on_remote" in config.keys():
-            self._ssh_delete_file_on_remote = config["ssh_delete_file_on_remote"]
-        else:
-            self._ssh_delete_file_on_remote = True
-        if "ssh_port" in config.keys():
-            self._ssh_port = config["ssh_port"]
-        else:
-            self._ssh_port = 22
-        if "ssh_continous_connection" in config.keys():
-            self._ssh_continous_connection = config["ssh_continous_connection"]
-        else:
-            self._ssh_continous_connection = False
+        self._ssh_delete_file_on_remote = config.get("ssh_delete_file_on_remote", True)
+        self._ssh_port = config.get("ssh_port", 22)
+        self._ssh_continous_connection = config.get("ssh_continous_connection", False)
         self._ssh_connection = None
         self._ssh_proxy_connection = None
         self._remote_flag = True
@@ -571,7 +553,7 @@ def _execute_remote_command(self, command: str) -> str:
         else:
             ssh = self._open_ssh_connection()
         stdin, stdout, stderr = ssh.exec_command(command)
-        warnings.warn(stderr.read().decode())
+        warnings.warn(message=stderr.read().decode(), stacklevel=2)
         output = stdout.read().decode()
         if not self._ssh_continous_connection:
             ssh.close()
@@ -592,7 +574,7 @@ def _get_remote_working_dir(self, working_directory: str) -> str:
             os.path.relpath(working_directory, self._ssh_local_path),
         )
 
-    def _create_remote_dir(self, directory: Union[str, List[str]]) -> None:
+    def _create_remote_dir(self, directory: Union[str, list[str]]) -> None:
         """
         Creates a remote directory on the SSH server.
 
@@ -628,7 +610,7 @@ def _transfer_data_to_remote(self, working_directory: str) -> None:
         )
         file_dict = {}
         new_dir_list = []
-        for p, folder, files in os.walk(working_directory):
+        for p, _folder, files in os.walk(working_directory):
             new_dir_list.append(
                 self._get_file_transfer(
                     file=p,

pysqa/base/validate.py

@@ -1,13 +1,13 @@
 import re
-from typing import Optional, Tuple, Union
+from typing import Optional, Union
 
 
 def check_queue_parameters(
     active_queue: Optional[dict] = None,
     cores: int = 1,
     run_time_max: Optional[int] = None,
     memory_max: Optional[int] = None,
-) -> Tuple[Union[float, int, None], Union[float, int, None], Union[float, int, None]]:
+) -> tuple[Union[float, int, None], Union[float, int, None], Union[float, int, None]]:
     """
     Check the parameters of a queue.