diff --git a/.gitignore b/.gitignore index b6e47617..8fb51adf 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,5 @@ dmypy.json # Pyre type checker .pyre/ +digitalpy/core/network/impl/stream_tcp_test.py +digitalpy/core/IAM/persistence/connections.json diff --git a/digitalpy/core/IAM/controllers/iam_users_controller.py b/digitalpy/core/IAM/controllers/iam_users_controller.py index abf9bfc5..f0a11ba6 100644 --- a/digitalpy/core/IAM/controllers/iam_users_controller.py +++ b/digitalpy/core/IAM/controllers/iam_users_controller.py @@ -4,19 +4,19 @@ from digitalpy.core.main.controller import Controller from digitalpy.core.domain.node import Node +from digitalpy.core.domain.domain.network_client import NetworkClient from digitalpy.core.zmanager.request import Request from digitalpy.core.zmanager.response import Response from digitalpy.core.zmanager.action_mapper import ActionMapper from digitalpy.core.digipy_configuration.configuration import Configuration from ..configuration.iam_constants import COMPONENT_NAME, CONNECTIONS_PERSISTENCE -from ..model.connection import Connection class IAMUsersController(Controller): def __init__(self, request: Request, response: Response, action_mapper: ActionMapper, configuration: Configuration): super().__init__(request, response, action_mapper, configuration) - def connection(self, logger, connection: Connection, **kargs): + def connection(self, logger, connection: NetworkClient, **kargs): """handle the case of a connection connection to any digitalpy service Args: diff --git a/digitalpy/core/IAM/model/connection.py b/digitalpy/core/IAM/model/connection.py index d52faa0d..84883d8f 100644 --- a/digitalpy/core/IAM/model/connection.py +++ b/digitalpy/core/IAM/model/connection.py @@ -3,8 +3,8 @@ class Connection(Node): def __init__(self, node_type = "connection", oid=None) -> None: super().__init__(node_type, oid=oid) - self._service_id = None - self._protocol = None + self._service_id: str = None + self._protocol: str = None @property def service_id(self): diff --git a/digitalpy/core/action_mapping.ini b/digitalpy/core/action_mapping.ini index d43cbb2f..bb32f5a2 100644 --- a/digitalpy/core/action_mapping.ini +++ b/digitalpy/core/action_mapping.ini @@ -23,10 +23,14 @@ __class = digitalpy.core.zmanager.impl.default_request.DefaultRequest [ActionMapper] ; this is the default action mapper __class = digitalpy.core.zmanager.impl.async_action_mapper.AsyncActionMapper +routing_subscriber_address = tcp://127.0.0.1:19030 +routing_publisher_address = tcp://127.0.0.1:19031 [AsyncActionMapper] ; this is a static reference to the async action mapper and should not be changed __class = digitalpy.core.zmanager.impl.async_action_mapper.AsyncActionMapper +routing_subscriber_address = tcp://127.0.0.1:19030 +routing_publisher_address = tcp://127.0.0.1:19031 [SyncActionMapper] ; this is a static reference to the sync action mapper and should not be changed @@ -51,11 +55,73 @@ __class = digitalpy.core.impl.default_file_logger.DefaultFileLogger [CotRouter] __class = FreeTAKServer.components.core.COT_Router.cot_router_facade.CotRouter +; the subject configuration [Subject] __class = digitalpy.core.zmanager.subject.Subject +frontend_pull_address = tcp://127.0.0.1:19030 +frontend_pub_address = tcp://127.0.0.1:19031 +backend_address = tcp://127.0.0.1:19031 +worker_count = 3 +; the integration manager configuration [IntegrationManager] __class = digitalpy.core.zmanager.integration_manager.IntegrationManager - +integration_manager_puller_protocol = tcp +integration_manager_puller_address = 127.0.0.1 +integration_manager_puller_port = 19033 +integration_manager_publisher_protocol = tcp +integration_manager_publisher_address = 127.0.0.1 +integration_manager_publisher_port = 19034 + +; the routing worker configuration [RoutingWorker] __class = digitalpy.core.zmanager.impl.default_routing_worker.DefaultRoutingWorker +server_address = tcp://127.0.0.1:19031 +integration_manager_address = tcp://127.0.0.1:19033 + +; the service manager configuration +[ServiceManager] +__class = digitalpy.core.service_management.controllers.service_management_main.ServiceManagementMain +subject_address = 127.0.0.1 +subject_port = 19030 +subject_protocol = tcp +integration_manager_address = 127.0.0.1 +integration_manager_port = 19034 +integration_manager_protocol = tcp +service_id = service_manager + +; the service configuration values +[Service] +subject_address = 127.0.0.1 +subject_port = 19030 +subject_protocol = tcp +integration_manager_address = 127.0.0.1 +integration_manager_port = 19034 +integration_manager_protocol = tcp + +; the service manager process controller class +[ServiceManagerProcessController] +__class = digitalpy.core.service_management.controllers.service_management_process_controller.ServiceManagementProcessController + +; the default tcp_network +[TCPNetwork] +__class = digitalpy.core.network.impl.network_tcp.TCPNetwork +client = DefaultClient + +[DefaultClient] +__class = digitalpy.core.domain.domain.network_client.NetworkClient + +; the default tracer exporter +[TracerExporter] +__class = opentelemetry.sdk.trace.export.ConsoleSpanExporter +;__class = FreeTAKServer.components.core.abstract_component.telemetry_exporter.ZMQExporter +;host = 127.0.0.1 +;port = 40033 + +; the processor mechanism for the tracer controller +[TracerProcessor] +__class = opentelemetry.sdk.trace.export.BatchSpanProcessor + +; the exporter mechanism for the tracer controller +[TracingProvider] +__class = digitalpy.core.telemetry.impl.opentel_tracing_provider.OpenTelTracingProvider diff --git a/digitalpy/core/component_management/component_management_facade.py b/digitalpy/core/component_management/component_management_facade_dev.py similarity index 100% rename from digitalpy/core/component_management/component_management_facade.py rename to digitalpy/core/component_management/component_management_facade_dev.py diff --git a/digitalpy/core/component_management/impl/component_registration_handler.py b/digitalpy/core/component_management/impl/component_registration_handler.py index 10508d91..f066d2b7 100644 --- a/digitalpy/core/component_management/impl/component_registration_handler.py +++ b/digitalpy/core/component_management/impl/component_registration_handler.py @@ -1,7 +1,7 @@ import importlib import os from pathlib import PurePath -from typing import List +from typing import Dict, List import pkg_resources from digitalpy.core.main.registration_handler import RegistrationHandler @@ -22,10 +22,12 @@ class ComponentRegistrationHandler(RegistrationHandler): """this class is used to manage component registration""" - registered_components = {} + registered_components: Dict[str, DefaultFacade] = {} pending_components = {} + component_index: Dict[str, Configuration] = {} + @staticmethod def clear(): ComponentRegistrationHandler.registered_components = {} @@ -109,8 +111,9 @@ def register_component( return False @staticmethod - def save_component(facade, component_name: str): + def save_component(facade: DefaultFacade, component_name: str): ComponentRegistrationHandler.registered_components[component_name] = facade + ComponentRegistrationHandler.component_index[component_name] = facade.get_manifest() @staticmethod def register_pending(component_name, config): @@ -118,7 +121,7 @@ def register_pending(component_name, config): facade_instance.register(config) @staticmethod - def validate_manifest(manifest: Configuration, component_name: str, component_facade) -> bool: + def validate_manifest(manifest: Configuration, component_name: str, component_facade) -> tuple[bool, bool]: #TODO: determine better way to inform the caller that the manifest is invalid """validate that the component is compatible with the current digitalpy version @@ -130,7 +133,7 @@ def validate_manifest(manifest: Configuration, component_name: str, component_fa ValueError: raised if the manifest section is missing from the manifest configuration Returns: - bool: whether the component is compatible with the current digitalpy installation + tuple[bool, bool]: whether the component is compatible with the current digitalpy installation and whether the component has any pending dependencies """ # retrieve the current digitalpy version based on the setup.py digitalpy_version = pkg_resources.require(DIGITALPY)[0].version @@ -148,7 +151,7 @@ def validate_manifest(manifest: Configuration, component_name: str, component_fa # validate the component name matches the name specified in the manifest if component_name != section[NAME]: - return False + return False, False # iterate the delimited version number and compare it to the digitalpy version for i in range(len(section[REQUIRED_ALFA_VERSION].split(VERSION_DELIMITER))): @@ -159,7 +162,7 @@ def validate_manifest(manifest: Configuration, component_name: str, component_fa elif int(digitalpy_version_number)==int(section[REQUIRED_ALFA_VERSION].split(VERSION_DELIMITER)[i]): continue else: - return False + return False, False # dont approve the manifest if the component has already been registered if ( @@ -167,7 +170,7 @@ def validate_manifest(manifest: Configuration, component_name: str, component_fa and section[VERSION] != ComponentRegistrationHandler.registered_components[component_name][VERSION] ): - return False + return False, False # dont approve the manifest if a component with the same name but a different ID already exists if ( @@ -175,7 +178,7 @@ def validate_manifest(manifest: Configuration, component_name: str, component_fa and ComponentRegistrationHandler.registered_components[component_name][ID] != section[ID] ): - return False + return False, False pending = False diff --git a/digitalpy/core/component_management/impl/default_facade.py b/digitalpy/core/component_management/impl/default_facade.py index 62e87db6..c25cbff1 100644 --- a/digitalpy/core/component_management/impl/default_facade.py +++ b/digitalpy/core/component_management/impl/default_facade.py @@ -1,12 +1,19 @@ +# pylint: disable=unused-argument +"""This is the default facade module. It is used to create a facade for a component. It is a simple example of a DigitalPyFacade. +""" +from types import ModuleType from digitalpy.core.main.controller import Controller from digitalpy.core.domain.node import Node from digitalpy.core.parsing.load_configuration import LoadConfiguration from digitalpy.core.digipy_configuration.impl.inifile_configuration import InifileConfiguration from digitalpy.core.zmanager.impl.default_action_mapper import DefaultActionMapper +from digitalpy.core.zmanager.request import Request +from digitalpy.core.zmanager.response import Response from digitalpy.core.main.object_factory import ObjectFactory from digitalpy.core.main.log_manager import LogManager from digitalpy.core.main.impl.default_file_logger import DefaultFileLogger -from digitalpy.core.main.controller import Controller +from digitalpy.core.digipy_configuration.configuration import Configuration + from digitalpy.core.telemetry.tracer import Tracer @@ -19,11 +26,11 @@ def __init__( log_file_path, component_name=None, type_mapping=None, - action_mapper: DefaultActionMapper = None, - base=object, - request=None, - response=None, - configuration=None, + action_mapper: DefaultActionMapper = None, # type: ignore + base=ModuleType, + request: Request = None, # type: ignore + response: Response = None, # type: ignore + configuration: Configuration = None, # type: ignore configuration_path_template=None, tracing_provider_instance=None, manifest_path=None, @@ -75,7 +82,7 @@ def __init__( self.component_name ) else: - self.tracer = None + self.tracer = None # type: ignore # load the manifest file as a configuration if manifest_path is not None: self.manifest = InifileConfiguration("") @@ -95,16 +102,19 @@ def __init__( else: self.config_loader = None - self.injected_values = {"logger": self.logger, "config_loader": self.config_loader, "tracer": self.tracer} + self.injected_values = { + "logger": self.logger, "config_loader": self.config_loader, "tracer": self.tracer} def initialize(self, request, response): super().initialize(request, response) self.request.set_sender(self.__class__.__name__) - def execute(self, method): + def execute(self, method=None) -> None: self.request.set_value("logger", self.logger) self.request.set_value("config_loader", self.config_loader) self.request.set_value("tracer", self.tracer) + if not method: + return try: if hasattr(self, method): # pass all request values as keyword arguments @@ -127,6 +137,7 @@ def public(func): facade methods being called directly. it's role is to inject internal attributes into the wrapped function which would generally be injected by the execute method""" + def wrapper(self, *args, **kwargs): # ensure that required values are passed by to controller # methods even if method isnt called through .execute method @@ -151,7 +162,7 @@ def register(self, config: InifileConfiguration, **kwargs): internal_config.add_configuration(self.internal_action_mapping_path) ObjectFactory.register_instance( f"{self.component_name.lower()}actionmapper", - self.base.ActionMapper( + self.base.ActionMapper( # type: ignore ObjectFactory.get_instance("event_manager"), internal_config, ), @@ -178,7 +189,8 @@ def _register_type_mapping(self): request.set_action("RegisterHumanToMachineMapping") # reverse the mapping and save the reversed mapping request.set_value( - "human_to_machine_mapping", {k: v for v, k in self.type_mapping.items()} + "human_to_machine_mapping", { + k: v for v, k in self.type_mapping.items()} ) actionmapper = ObjectFactory.get_instance("SyncActionMapper") @@ -187,3 +199,16 @@ def _register_type_mapping(self): def accept_visitor(self, node: Node, visitor, **kwargs): return node.accept_visitor(visitor) + + def __setstate__(self, state: dict) -> None: + from .. import base + self.__dict__ = state + if "base" in state: + self.base = base + + def __getstate__(self) -> dict: + tmp = self.__dict__ + set_base = tmp.get("base", None) + if set_base is not None: + tmp["base"] = True + return tmp diff --git a/digitalpy/core/cot_management/controllers/cot_management_general_controller.py b/digitalpy/core/cot_management/controllers/cot_management_general_controller.py deleted file mode 100644 index 29097674..00000000 --- a/digitalpy/core/cot_management/controllers/cot_management_general_controller.py +++ /dev/null @@ -1,53 +0,0 @@ -####################################################### -# -# core_name_general_controller.py -# Python implementation of the Class CoreNameGeneralController -# Generated by Enterprise Architect -# Created on: 16-Dec-2022 10:56:05 AM -# Original author: Giu Platania -# -####################################################### -from Catalog.Implementation.Libraries.Digitalpy.digitalpy.Async.routing.controller import Controller - -class COTManagementGeneralController(Controller): -# default constructor def __init__(self): - - def __init__(Request, Response, ActionMapper, Configuration): - pass - - def execute( = None): - pass - - def serialize_cot_management(): - """this is the general method used to serialize the component to a given format - """ - pass - - def build_drop_point_object(): - """instantiate domain object with drop point structure""" - pass - - def cot__share_privately(): - """Send COT to specific client""" - pass - - def cot_record_in_db(): - """record all the traffic in the Database""" - pass - - def cot__broadcast(): - """send the CoT to all connected clients""" - pass - - def medevac__receive(): - """Receive a Medical Evaluation send by another EUD trough the server""" - pass - - def web_ui_manage_presence(): - """Using this function, the user can create a custom team member (friendly Dot) on the ATAK map, who will however appear like a real TAK user.""" - pass - - def medevac__send(): - """send a MedEvac form to another EUD connected to the server""" - pass - diff --git a/digitalpy/core/cot_management/cot_management_facade.py b/digitalpy/core/cot_management/cot_management_facade.py deleted file mode 100644 index 94a487f7..00000000 --- a/digitalpy/core/cot_management/cot_management_facade.py +++ /dev/null @@ -1,29 +0,0 @@ -from Catalog.Data.Domain.FTS_Model.Facade import Facade - -class COTManagementFacade(Facade, Facade): - """Facade class for the this component. Responsible for handling all public - routing. Forwards all requests to the internal router. - WHY - - """ - -# default constructor def __init__(self): - - def __init__(self): - self.build_drop_point_object = COTManagementGeneralController() - self.cot__share_privately = COTManagementGeneralController() - self.cot_record_in_db = COTManagementGeneralController() - self.cot__broadcast = COTManagementGeneralController() - self.medevac__receive = COTManagementGeneralController() - self.web_ui_manage_presence = COTManagementGeneralController() - self.medevac__send = COTManagementGeneralController() - - diff --git a/digitalpy/core/digipy_configuration/configuration.py b/digitalpy/core/digipy_configuration/configuration.py index a02a88ea..56296cd2 100644 --- a/digitalpy/core/digipy_configuration/configuration.py +++ b/digitalpy/core/digipy_configuration/configuration.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from typing import Any class Configuration(ABC): """Implementations of Configuration give access to the application @@ -27,7 +28,7 @@ def add_configuration(self, name: str, process_values: bool =True): """Parses the given configuration and merges it with already added configurations.""" @abstractmethod - def get_sections(self)-> list: + def get_sections(self)-> list[str]: """Get all section names.""" @@ -47,7 +48,7 @@ def has_value(self, key: str, section: str) -> bool: @abstractmethod - def get_value(self, key: str, section: str): + def get_value(self, key: str, section: str) -> Any: """Get a configuration value.""" diff --git a/digitalpy/core/digipy_configuration/impl/inifile_configuration.py b/digitalpy/core/digipy_configuration/impl/inifile_configuration.py index 2125529a..3c653fb1 100644 --- a/digitalpy/core/digipy_configuration/impl/inifile_configuration.py +++ b/digitalpy/core/digipy_configuration/impl/inifile_configuration.py @@ -47,7 +47,7 @@ def add_configuration(self, name, process_values=True): return if not os.path.exists(filename): - raise ValueError("Could not find configuration file") + raise ValueError("Could not find configuration file "+str(filename)) self.__added_files.append(filename) result = self.process_file(filename, self.config_array, self.contained_files) @@ -186,7 +186,7 @@ def get_section(self, section: str, include_meta=False) -> dict: return { key: val for key, val in self.config_array[lookup_entry[0]].items() - if re.match("/^__/", key) + if not re.match("/^__/", key) } def _lookup(self, section, key=""): diff --git a/digitalpy/core/domain/base/__init__.py b/digitalpy/core/domain/base/__init__.py new file mode 100644 index 00000000..691e00f1 --- /dev/null +++ b/digitalpy/core/domain/base/__init__.py @@ -0,0 +1,4 @@ +"""this module contains all the supporting components without business logic +it should also be noted that the component action mapper must be exposed as action +mapper.""" +from .domain_action_mapper import DomainActionMapper as ActionMapper diff --git a/digitalpy/core/domain/base/domain_action_mapper.py b/digitalpy/core/domain/base/domain_action_mapper.py new file mode 100644 index 00000000..fbfb7429 --- /dev/null +++ b/digitalpy/core/domain/base/domain_action_mapper.py @@ -0,0 +1,8 @@ +from digitalpy.core.zmanager.impl.default_action_mapper import DefaultActionMapper + + +class DomainActionMapper(DefaultActionMapper): + """this is the Domain component action mapper, each component + must have its own action mapper to be loaded with the internal + action mapping configuration and to be used by the facade for + internal routing""" diff --git a/digitalpy/core/domain/base/domain_health_check.py b/digitalpy/core/domain/base/domain_health_check.py new file mode 100644 index 00000000..52b5a735 --- /dev/null +++ b/digitalpy/core/domain/base/domain_health_check.py @@ -0,0 +1,6 @@ +from digitalpy.component.impl.default_health_check import DefaultHealthCheckController + + +class DomainHealthCheck(DefaultHealthCheckController): + """this class only inherits from the default health check controller so that the internal \ + action mapper maps to a local controller""" diff --git a/digitalpy/core/domain/base/domain_metrics_controller.py b/digitalpy/core/domain/base/domain_metrics_controller.py new file mode 100644 index 00000000..1129a90b --- /dev/null +++ b/digitalpy/core/domain/base/domain_metrics_controller.py @@ -0,0 +1,24 @@ +"""this module contains only the DomainMetricsController class""" +from digitalpy.core.main.impl.default_meter_controller import MeterController +from ..configuration.domain_constants import COMPONENT_NAME, METRICS_ADDRESS + + +class DomainMetricsController(MeterController): + """the metrics controller implementation for the domain + component.""" + + def __init__(self, request, response, action_mapper, configuration): + super().__init__( + COMPONENT_NAME, + METRICS_ADDRESS, + request, + response, + action_mapper, + configuration, + ) + + def test_metrics(self): + """test the metrics functionality of the domain metrics controller""" + + counter = self.meter.create_counter("test", "test", "1") + counter.increment(2, {"abc": "123"}) diff --git a/digitalpy/core/domain/builder.py b/digitalpy/core/domain/builder.py new file mode 100644 index 00000000..08096827 --- /dev/null +++ b/digitalpy/core/domain/builder.py @@ -0,0 +1,47 @@ +from abc import ABC + +from digitalpy.core.main.controller import Controller +from digitalpy.core.zmanager.request import Request +from digitalpy.core.zmanager.response import Response +from digitalpy.core.zmanager.action_mapper import ActionMapper +from digitalpy.core.digipy_configuration.configuration import Configuration +from digitalpy.core.parsing.load_configuration import LoadConfiguration + +class Builder(Controller): + """manage operations related to domain""" + def __init__(self, request: Request, response: Response, sync_action_mapper: ActionMapper, configuration: Configuration): + super().__init__(request, response, sync_action_mapper, configuration) + self.result: any = None + + def initialize(self, request, response): + super().initialize(request, response) + + def execute(self, method=None): + getattr(self, method)(**self.request.get_values()) + return self.response + + def build_empty_object(self, config_loader): + raise NotImplementedError + + def add_object_data(self, mapped_object): + raise NotImplementedError + + def get_result(self): + return self.result + + def _create_model_object(self, configuration, extended_domain={}, *args, **kwargs): + self.request.set_value("configuration", configuration) + + self.request.set_value("extended_domain", extended_domain) + + self.request.set_value( + "source_format", self.request.get_value("source_format") + ) + self.request.set_value("target_format", "node") + + response = self.execute_sub_action("CreateNode") + + model_object = response.get_value("model_object") + + return model_object + \ No newline at end of file diff --git a/digitalpy/core/domain/configuration/__init__.py b/digitalpy/core/domain/configuration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/digitalpy/core/domain/configuration/domain_constants.py b/digitalpy/core/domain/configuration/domain_constants.py new file mode 100644 index 00000000..e68a073f --- /dev/null +++ b/digitalpy/core/domain/configuration/domain_constants.py @@ -0,0 +1,54 @@ +import pathlib + +COMPONENT_NAME = "Domain" + +CURRENT_COMPONENT_PATH = pathlib.Path(__file__).parent.parent.absolute() + +LOGGING_CONFIGURATION_PATH = str( + pathlib.PurePath(CURRENT_COMPONENT_PATH, "configuration/logging.conf") +) + +ACTION_MAPPING_PATH = str( + pathlib.PurePath( + CURRENT_COMPONENT_PATH, "configuration/external_action_mapping.ini" + ) +) + +INTERNAL_ACTION_MAPPING_PATH = str( + pathlib.PurePath( + CURRENT_COMPONENT_PATH, "configuration/internal_action_mapping.ini" + ) +) + +SERIALIZATION_BUSINESS_RULES_PATH = str( + pathlib.PurePath( + CURRENT_COMPONENT_PATH, + "configuration/business_rules/serialization_business_rules.json", + ) +) + +DOMAIN_BUSINESS_RULES_PATH = str( + pathlib.PurePath( + CURRENT_COMPONENT_PATH, + "configuration/business_rules/domain_business_rules.json", + ) +) + +METRICS_ADDRESS = str( + pathlib.PurePath( + CURRENT_COMPONENT_PATH, + "configuration/metrics.txt", + ) +) + +MANIFEST_PATH = str( + pathlib.PurePath(CURRENT_COMPONENT_PATH, "configuration/manifest.ini") +) + +# TODO this path shouldn't be hardcoded, +# find way to change this to a configured value. +LOG_FILE_PATH = str( + pathlib.PurePath(CURRENT_COMPONENT_PATH, "logs") +) + +BASE_OBJECT_NAME = "Event" diff --git a/digitalpy/core/domain/configuration/external_action_mapping.ini b/digitalpy/core/domain/configuration/external_action_mapping.ini new file mode 100644 index 00000000..d07e7b58 --- /dev/null +++ b/digitalpy/core/domain/configuration/external_action_mapping.ini @@ -0,0 +1,32 @@ +[actionmapping] +??GetNodeParent = digitalpy.core.domain.domain_facade.Domain.get_parent + +??CreateNode = digitalpy.core.domain.domain_facade.Domain.create_node + +??HealthCheck = digitalpy.core.domain.domain_facade.Domain.get_health + +??GetMetrics = digitalpy.core.domain.domain_facade.Domain.get_metrics + +??TestMetrics = digitalpy.core.domain.domain_facade.Domain.test_metrics + +??GetTraces = digitalpy.core.domain.domain_facade.Domain.get_traces + +??TestTracing = digitalpy.core.domain.domain_facade.Domain.test_tracing + +[Request] +__class = digitalpy.core.zmanager.impl.default_request.DefaultRequest + +[ActionMapper] +__class = digitalpy.core.zmanager.impl.default_action_mapper.DefaultActionMapper + +[event_manager] +__class = digitalpy.core.main.impl.default_event_manager.DefaultEventManager + +[Response] +__class = digitalpy.core.zmanager.impl.default_response.DefaultResponse + +[Domain] +__class = digitalpy.core.domain.domain_facade.Domain + +[DomainMetrics] +__class = digitalpy.core.component_management.impl.opentel_metrics.OpenTelMetrics \ No newline at end of file diff --git a/digitalpy/core/domain/configuration/internal_action_mapping.ini b/digitalpy/core/domain/configuration/internal_action_mapping.ini new file mode 100644 index 00000000..6b51054e --- /dev/null +++ b/digitalpy/core/domain/configuration/internal_action_mapping.ini @@ -0,0 +1,35 @@ +[actionmapping] +??GetNodeParent = FreeTAKServer.components.core.domain.controllers.domain.Domain.get_parent + +??CreateNode = FreeTAKServer.components.core.domain.controllers.domain.Domain.create_node + +; serialize a source format to a target format +??Serialize = FreeTAKServer.components.core.domain.controllers.dict_to_node_controller.DictToNodeController.convert_dict_to_node + +??DictToNode = FreeTAKServer.components.core.domain.controllers.dict_to_node_controller.DictToNodeController.convert_dict_to_node + +??XMLToNode = FreeTAKServer.components.core.domain.controllers.serialization.xml_serialization.xml_serialization_orchestrator.XMLSerializationOrchestrator.xml_to_node + +??NodeToXML = = FreeTAKServer.components.core.domain.controllers.serialization.xml_serialization.xml_serialization_orchestrator.XMLSerializationOrchestrator.nod_to_xml + +??HealthCheck = FreeTAKServer.components.core.domain.base.domain_health_check.DomainHealthCheck.get_health + +??GetMetrics = FreeTAKServer.components.core.domain.base.domain_metrics_controller.DomainMetricsController.get_metrics + +??TestMetrics = FreeTAKServer.components.core.domain.base.domain_metrics_controller.DomainMetricsController.test_metrics + +??GetTraces = FreeTAKServer.components.core.domain.base.domain_tracing_controller.DomainTracingController.get_traces + +??TestTracing = FreeTAKServer.components.core.domain.base.domain_tracing_controller.DomainTracingController.test_tracing + +[Request] +__class = digitalpy.core.zmanager.impl.default_request.DefaultRequest + +[ActionMapper] +__class = digitalpy.core.zmanager.impl.default_action_mapper.DefaultActionMapper + +[event_manager] +__class = digitalpy.core.main.impl.default_event_manager.DefaultEventManager + +[Response] +__class = digitalpy.core.zmanager.impl.default_response.DefaultResponse \ No newline at end of file diff --git a/digitalpy/core/domain/configuration/logging.conf b/digitalpy/core/domain/configuration/logging.conf new file mode 100644 index 00000000..6ad96b69 --- /dev/null +++ b/digitalpy/core/domain/configuration/logging.conf @@ -0,0 +1,32 @@ +[loggers] +keys=root,domain + +[handlers] +keys=stream_handler,fileHandler + +[formatters] +keys=formatter + +[logger_root] +level=DEBUG +handlers=fileHandler + +[logger_domain] +level=DEBUG +qualname=domain +handlers=fileHandler + +[handler_stream_handler] +class=StreamHandler +level=DEBUG +formatter=formatter +args=(sys.stderr,) + +[handler_fileHandler] +class=FileHandler +level=DEBUG +formatter=formatter +args=('%(logfilename)s',) + +[formatter_formatter] +format=%(asctime)s %(name)-12s %(levelname)-8s %(message)s \ No newline at end of file diff --git a/digitalpy/core/domain/configuration/manifest.ini b/digitalpy/core/domain/configuration/manifest.ini new file mode 100644 index 00000000..b13f0d49 --- /dev/null +++ b/digitalpy/core/domain/configuration/manifest.ini @@ -0,0 +1,4 @@ +[DomainManifest] +name=domain +requiredAlfaVersion=0.2.7 +version=0.1 \ No newline at end of file diff --git a/digitalpy/core/domain/controllers/domain_controller.py b/digitalpy/core/domain/controllers/domain_controller.py index 5505e0b1..987c055f 100644 --- a/digitalpy/core/domain/controllers/domain_controller.py +++ b/digitalpy/core/domain/controllers/domain_controller.py @@ -1,27 +1,200 @@ -####################################################### -# -# core_name_controller.py -# Python implementation of the Class CoreNameRulesController -# Generated by Enterprise Architect -# Created on: 16-Dec-2022 10:56:02 AM -# Original author: Giu Platania -# -####################################################### -from Catalog.Implementation.Libraries.Digitalpy.digitalpy.Async.logic.impl.default_business_rule_controller import DefaultBusinessRuleController - -class DomainController(DefaultBusinessRuleController): - """contains all the business logic of this core package - """ -# default constructor def __init__(self): - - def __init__(): - pass - - def execute( = None): - pass - - def parse_domain(): - """Creates the model object outline and passes it to the parser to fill the model - object with the xml data - """ - pass +"""this module is responsible for handling all domain interactions""" +import uuid +from types import ModuleType +from typing import Any, List, Optional, Type +from digitalpy.core.zmanager.action_mapper import ActionMapper +from digitalpy.core.digipy_configuration.configuration import Configuration +from digitalpy.core.parsing.load_configuration import ModelConfiguration as LConfiguration + +from digitalpy.core.zmanager.request import Request +from digitalpy.core.zmanager.response import Response +from digitalpy.core.domain.node import Node +from digitalpy.core.main.controller import Controller +from digitalpy.core.main.object_factory import ObjectFactory + +from digitalpy.core.domain import domain + + +class DomainController(Controller): + """this is the controller for the domain component, + it is responsible for handling all domain interactions""" + + def __init__( + self, + request: Request, + response: Response, + domain_action_mapper: ActionMapper, + configuration: Configuration, + **kwargs, # pylint: disable=unused-argument + ): + super().__init__(request, response, domain_action_mapper, configuration) + self.domain = domain + + def execute(self, method=None): + return getattr(self, method)(**self.request.get_values()) + + def add_child(self, node: Node, child: Node, **kwargs) -> None: # pylint: disable=unused-argument + """add a child to a node + + Args: + node (Node): the origin node + child (Node): the node to be added as the original node + + Returns: + _type_: _description_ + """ + return node.add_child(child) + + def create_node(self, configuration: LConfiguration, object_class_name: str, id: str = None, **kwargs) -> Node: + """this method creates a new node object + + Args: + configuration (LConfiguration): _description_ + object_class_name (str): _description_ + id (str): the id of the created node + """ + if id is None: + id = str(uuid.uuid1()) + # allow the domain to be extended + domaindict = self._extend_domain( + self.domain, kwargs.get('extended_domain', {})) + # retrieve the original object class + object_class: type[Node] = domaindict[object_class_name] + # instantiate an oid for the instance + oid = ObjectFactory.get_instance( + "ObjectId", {"id": id, "type": object_class_name}) + # instantiate the object class + object_class_instance = object_class( + model_configuration=configuration, model=domaindict, oid=oid) + # set the module object + self.response.set_value("model_object", object_class_instance) + return object_class_instance + + def _extend_domain(self, domain: ModuleType, extended_domain: dict) -> dict: + """this method is responsible for adding domain extensions from a given component + + Args: + domain (ModuleType): the base domain package + extended_domain (dict): the updated domain package + + Returns: + ModuleType: an updated domain + """ + domaindict = domain.__dict__.copy() + for key, value in extended_domain.items(): + domaindict[key] = value + return domaindict + + def delete_child(self, node: Node, child_id: str, **kwargs): + """delete a child node + + Args: + node (Node): the node from which to remove the child + child_id (str): the id of the child to be deleted + + Returns: + None + """ + return node.delete_child(child_id) + + def get_children_ex( + self, + id, + node: Node, + children_type, + values, + properties, + use_regex=True, + **kwargs, + ): + self.response.set_value( + "children", + node.get_children_ex( + id, node, children_type, values, properties, use_regex + ), + ) + + def get_first_child(self, node: Node, child_type: Type[Node], # pylint: disable=unused-argument + values: "dict[str, Any]", properties: "dict[str, Any]", + use_regex: bool = True, **kwargs) -> Optional[Node]: + """Returns the first child of the given node that matches the given child type, + values, and properties. + + Args: + node (Node): The node to get the first child of. + child_type (Type[Node]): The type of the child to find. + values (dict[str, Any]): The values the child must have. + properties (dict[str, Any]): The properties the child must have. + use_regex (bool, optional): Whether to use regular expressions to match values and + properties. Defaults to True. + **kwargs: Additional keyword arguments. + + Returns: + Optional[Node]: The first child that matches the given child type, values, + and properties, or None if no such child is found. + """ + self.response.set_value("first_child", node.get_first_child( + child_type, values, properties, use_regex)) + + def get_next_sibling(self, node: Node, **kwargs) -> Optional[Node]: + """Returns the next sibling of the given node. + + Args: + node (Node): The node to get the next sibling of. + **kwargs: Additional keyword arguments. + + Returns: + Optional[Node]: The next sibling of the given node, or None if the node has no next sibling. + """ + self.response.set_value("next_sibling", node.get_next_sibling()) + + def get_num_children(self, node: Node, children_type: Optional[Type[Node]] = None, **kwargs) -> int: + """Returns the number of children the given node has. + + Args: + node (Node): The node to get the number of children of. + children_type (Optional[Type[Node]], optional): The type of children to count. If not specified, all children are counted. Defaults to None. + **kwargs: Additional keyword arguments. + + Returns: + int: The number of children the given node has. + """ + self.response.set_value( + "num_children", node.get_num_children(children_type)) + + def get_num_parents(self, node: Node, parent_types: Optional[List[Type[Node]]] = None, **kwargs) -> int: + """Returns the number of parents the given node has. + + Args: + node (Node): The node to get the number of parents of. + parent_types (Optional[List[Type[Node]]], optional): The types of parents to count. If not specified, all parents are counted. Defaults to None. + **kwargs: Additional keyword arguments. + + Returns: + int: The number of parents the given node has. + """ + self.response.set_value( + "num_parents", node.get_num_parents(parent_types)) + + def get_previous_sibling(self, node: Node) -> Optional[Node]: + """Returns the previous sibling of the given node. + + Args: + node (Node): The node to get the previous sibling of. + + Returns: + Optional[Node]: The previous sibling of the given node, or None if the node has no previous sibling. + """ + self.response.set_value( + "previous_sibling", node.get_previous_sibling()) + + def get_parent(self, node: Node) -> Optional[Node]: + """Returns the parent of the given node. + + Args: + node (Node): The node to get the parent of. + + Returns: + Optional[Node]: The parent of the given node, or None if the node has no parent. + """ + self.response.set_value("parent", node.get_parent()) diff --git a/digitalpy/core/domain/domain/__init__.py b/digitalpy/core/domain/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/digitalpy/core/domain/domain/network_client.py b/digitalpy/core/domain/domain/network_client.py new file mode 100644 index 00000000..a9cfd095 --- /dev/null +++ b/digitalpy/core/domain/domain/network_client.py @@ -0,0 +1,106 @@ +import uuid +from digitalpy.core.domain.node import Node +from digitalpy.core.domain.object_id import ObjectId + +from ...network.domain.client_status import ClientStatus + + +class NetworkClient(Node): + """this class represents a network client in the digitalpy framework, + it is the base network client implementation and can be expanded""" + + def __init__(self, model_configuration=None, model=None, oid=None, + node_type="network_client") -> None: + # all network clients should have a default node id + if oid is None: + oid = ObjectId(node_type, str(uuid.uuid4())) + super().__init__(node_type, model_configuration=model_configuration, + model=model, oid=oid) # type: ignore + # the id of the client + self._id: bytes + # the status of the client + self._status: ClientStatus = ClientStatus.CONNECTING + # id of the related service + self._service_id: str + # the protocol used by the client + self._protocol: str + + @property + def protocol(self) -> str: + """get the protocol of the client + + Returns: + str: the protocol of the client + """ + return self._protocol + + @protocol.setter + def protocol(self, protocol: str): + """set the protocol of the client + + Args: + protocol (str): the protocol of the client + """ + if not isinstance(protocol, str): + raise TypeError("'protocol' must be an instance of str") + self._protocol = protocol + + @property + def service_id(self) -> str: + """get the service id of the client + + Returns: + str: the service id of the client + """ + return self._service_id + + @service_id.setter + def service_id(self, service_id: str): + """set the service id of the client + + Args: + service_id (str): the service id of the client + """ + if not isinstance(service_id, str): + raise TypeError("'service_id' must be an instance of str") + self._service_id = service_id + + @property + def status(self) -> ClientStatus: + """get the status of the client + + Returns: + str: the status of the client + """ + return self._status + + @status.setter + def status(self, status: ClientStatus): + """set the status of the client + + Args: + status (str): the status of the client + """ + if status not in ClientStatus: + raise ValueError("'status' must be a valid ClientStatus") + self._status = status + + @property + def id(self) -> bytes: + """get the id of the client + + Returns: + bytes: the id of the client + """ + return self._id + + @id.setter + def id(self, id: bytes): + """set the id of the client + + Args: + id (bytes): the id of the client + """ + if not isinstance(id, bytes): + raise TypeError("'id' must be an instance of int") + self._id = id diff --git a/digitalpy/core/domain/domain/service_health.py b/digitalpy/core/domain/domain/service_health.py new file mode 100644 index 00000000..1a64b8f2 --- /dev/null +++ b/digitalpy/core/domain/domain/service_health.py @@ -0,0 +1,125 @@ +from datetime import datetime +from digitalpy.core.domain.node import Node +from digitalpy.core.health.domain.service_health_category import ServiceHealthCategory + + +class ServiceHealth(Node): + """service health class""" + + def __init__(self, model_configuration=None, model=None, oid=None, + node_type="service_health") -> None: + super().__init__(node_type, model_configuration=model_configuration, + model=model, oid=oid) # type: ignore + # id of the service + self._service_id: str + # the status of the service + self._status: ServiceHealthCategory + # the time that the health was recorded + self._timestamp: datetime + # the percentage of errors per request + self._error_percentage: float + + @property + def average_request_time(self) -> float: + """get the average request time of the service + + Returns: + float: the average request time of the service + """ + return self._average_request_time + + @average_request_time.setter + def average_request_time(self, average_request_time: float): + """set the average request time of the service + + Args: + average_request_time (float): the average request time of the service + """ + if not isinstance(average_request_time, float) and not isinstance(average_request_time, int): + raise TypeError( + "'average_request_time' must be an instance of float") + self._average_request_time = float(average_request_time) + + @property + def error_percentage(self) -> float: + """get the error percentage of the service + + Returns: + float: the error percentage of the service + """ + return self._error_percentage + + @error_percentage.setter + def error_percentage(self, error_percentage: float): + """set the error percentage of the service + + Args: + error_percentage (float): the error percentage of the service + """ + if not isinstance(error_percentage, float) and not isinstance(error_percentage, int): + raise TypeError("'error_percentage' must be an instance of float") + self._error_percentage = float(error_percentage) + + @property + def service_id(self) -> str: + """get the service id of the service + + Returns: + str: the service id of the service + """ + return self._service_id + + @service_id.setter + def service_id(self, service_id: str): + """set the service id of the service + + Args: + service_id (str): the service id of the service + """ + if not isinstance(service_id, str): + raise TypeError("'service_id' must be an instance of str") + self._service_id = service_id + + @property + def status(self) -> ServiceHealthCategory: + """get the status of the service + + Returns: + str: the status of the service + """ + return self._status + + @status.setter + def status(self, status: ServiceHealthCategory): + """set the status of the service + + Args: + status (str): the status of the service + """ + if not isinstance(status, ServiceHealthCategory): + raise TypeError( + "'status' must be an instance of ServiceHealthCategory") + self._status = status + + @property + def timestamp(self) -> datetime: + """get the timestamp of the service + + Returns: + datetime: the timestamp of the service + """ + return self._timestamp + + @timestamp.setter + def timestamp(self, timestamp: datetime): + """set the timestamp of the service + + Args: + timestamp (datetime): the timestamp of the service + """ + if not isinstance(timestamp, datetime): + raise TypeError("'timestamp' must be an instance of datetime") + self._timestamp = timestamp + + def __str__(self) -> str: + return f"ServiceHealth(service_id={self.service_id}, status={self.status}, timestamp={self.timestamp}, error_percentage={self.error_percentage}, average_request_time={self.average_request_time})" diff --git a/digitalpy/core/domain/domain_facade.py b/digitalpy/core/domain/domain_facade.py new file mode 100644 index 00000000..940b91d5 --- /dev/null +++ b/digitalpy/core/domain/domain_facade.py @@ -0,0 +1,83 @@ +"""This module contains the facade class for the domain component +""" +from digitalpy.core.component_management.impl.default_facade import DefaultFacade +from digitalpy.core.domain.configuration.domain_constants import ( + ACTION_MAPPING_PATH, + LOGGING_CONFIGURATION_PATH, + INTERNAL_ACTION_MAPPING_PATH, + MANIFEST_PATH, + LOG_FILE_PATH, +) + +from .controllers.domain_controller import DomainController +from . import base # pylint: disable=no-name-in-module + + +class Domain(DefaultFacade): + """This is the facade class for the domain component, it is responsible + for handling all public routing and forwards all requests to the internal routing + """ + + def __init__( + self, + sync_action_mapper, + request, + response, + configuration, + tracing_provider_instance=None, + ): + super().__init__( + # the path to the external action mapping + action_mapping_path=ACTION_MAPPING_PATH, + # the path to the internal action mapping + internal_action_mapping_path=INTERNAL_ACTION_MAPPING_PATH, + # the path to the logger configuration + logger_configuration=LOGGING_CONFIGURATION_PATH, + # the package containing the base classes + base=base, + # the component specific action mapper (passed by constructor) + action_mapper=sync_action_mapper, + # the request object (passed by constructor) + request=request, + # the response object (passed by constructor) + response=response, + # the configuration object (passed by constructor) + configuration=configuration, + # log file path + log_file_path=LOG_FILE_PATH, + # the tracing provider used + tracing_provider_instance=tracing_provider_instance, + # the path to the manifest file + manifest_path=MANIFEST_PATH, + ) + self.domain_controller = DomainController( + request, response, sync_action_mapper, configuration) + + def initialize(self, request, response): + super().initialize(request, response) + self.domain_controller.initialize(request, response) + + def execute(self, method=None): + """this method executes the action requested + """ + try: + if method is not None and hasattr(self, method): + getattr(self, method)(**self.request.get_values()) + else: + self.request.set_value("logger", self.logger) + self.request.set_value("config_loader", self.config_loader) + self.request.set_value("tracer", self.tracer) + response = self.execute_sub_action(self.request.get_action()) + self.response.set_values(response.get_values()) + except Exception as e: + self.logger.fatal(str(e)) + + @DefaultFacade.public + def create_node(self, *args, **kwargs): + """this method creates a new node object""" + self.domain_controller.create_node(*args, **kwargs) + + @DefaultFacade.public + def get_node_parent(self, *args, **kwargs): + """this method gets the parent of a node""" + self.domain_controller.get_parent(*args, **kwargs) diff --git a/digitalpy/core/domain/node.py b/digitalpy/core/domain/node.py index 89d7369d..d9f4b0f5 100644 --- a/digitalpy/core/domain/node.py +++ b/digitalpy/core/domain/node.py @@ -1,8 +1,8 @@ import uuid import re -from typing import Dict, Any +from typing import Dict, Any, Union from digitalpy.core.persistence.impl.default_persistent_object import DefaultPersistentObject -from digitalpy.core.parsing.load_configuration import Configuration +from digitalpy.core.parsing.load_configuration import ModelConfiguration from digitalpy.core.domain.object_id import ObjectId from digitalpy.core.persistence.persistent_object import PersistentObject from digitalpy.core.persistence.persistent_object_proxy import PersistentObjectProxy @@ -30,9 +30,9 @@ class Node(DefaultPersistentObject): def __init__( self, - node_type, - configuration: Configuration = Configuration(), - model = None, + node_type="node", + model_configuration: Union[ModelConfiguration, None] = None, + model=None, oid: ObjectId = None, initial_data=None, ) -> None: @@ -46,18 +46,34 @@ def __init__( initial_data (_type_, optional): _description_. Defaults to None. """ super().__init__(oid, initial_data) + if model_configuration is None: + model_configuration = ModelConfiguration() self._children: Dict[str, Node] = {} self._parents: Dict[str, Node] = {} self._depth = -1 self._path = "" + # any extended domain objects which are not defined in the domain + self._extended = {} # default the elements to an empty dictionary if the class configuration doesn't exist - self._relationship_definition = configuration.elements.get(self.__class__.__name__, None) - + self._relationship_definition = model_configuration.elements.get( + self.__class__.__name__, None) + # check that the value of _relationship_definition is not none if self._relationship_definition != None: - self._add_relationships(configuration, model) + self._add_relationships(model_configuration, model) + + @property + def extended(self) -> dict: + """this property returns the extended domain objects which are not defined in the domain + """ + return self._extended + + @extended.setter + def extended(self, value: dict): + """this property sets the extended domain objects which are not defined in the domain""" + self._extended = value - def _add_relationships(self, configuration: Configuration, model) -> None: + def _add_relationships(self, configuration: ModelConfiguration, model) -> None: for ( relationship_name, relationship_def, @@ -295,7 +311,8 @@ def get_first_parent( as regular expressions or not (default: _True_) @return Node instance or None. """ - parents = self.get_parents_ex(None, role, type, values, properties, use_regex) + parents = self.get_parents_ex( + None, role, type, values, properties, use_regex) if len(parents) > 0: return parents[0] @@ -398,7 +415,8 @@ def add_node( else: None - result_2 = other.add_node(self, this_role, force_set, track_change, False) + result_2 = other.add_node( + self, this_role, force_set, track_change, False) return result_1 & result_2 @@ -693,7 +711,8 @@ def get_value(self, name: Any) -> Any: self.relation_states[name] = self.RELATION_STATE_INITIALIZING mapper = self.get_mapper() - all_relatives = mapper.load_relation([self], name, BuildDepth.PROXIES_ONLY) + all_relatives = mapper.load_relation( + [self], name, BuildDepth.PROXIES_ONLY) oid_str = str(self.get_oid()) if oid_str in all_relatives: relatives = all_relatives[oid_str] @@ -741,7 +760,8 @@ def load_children( self.load_relations([role], build_depth) else: - self.load_relations(self.get_possible_children().keys(), build_depth) + self.load_relations( + self.get_possible_children().keys(), build_depth) def load_parents( self, role: Any = None, build_depth: Any = BuildDepth.SINGLE @@ -757,7 +777,8 @@ def load_parents( self.load_relations([role], build_depth) else: - self.load_relations(self.get_possible_parents().keys(), build_depth) + self.load_relations( + self.get_possible_parents().keys(), build_depth) def load_relations(self, roles: list, build_depth: Any = BuildDepth.SINGLE) -> Any: """Load all objects in the given set of relations""" @@ -776,7 +797,8 @@ def load_relations(self, roles: list, build_depth: Any = BuildDepth.SINGLE) -> A if isinstance(cur_relative, PersistentObjectProxy): # resolve proxies cur_relative.resolve(build_depth) - relatives.append(cur_relative.get_real_subject()) + relatives.append( + cur_relative.get_real_subject()) else: relatives.append(cur_relative) @@ -784,7 +806,8 @@ def load_relations(self, roles: list, build_depth: Any = BuildDepth.SINGLE) -> A # otherwise load the objects directly else: mapper = self.get_mapper() - all_relatives = mapper.load_relation([self], cur_role, build_depth) + all_relatives = mapper.load_relation( + [self], cur_role, build_depth) oid_str = self.get_o_i_d().__toString() if oid_str in all_relatives: relatives = all_relatives[oid_str] @@ -852,10 +875,12 @@ def merge_values(self, object: PersistentObject) -> Any: existing_value = self.parent_get_value_method.invoke_args( self, [value_name] ) - new_value = self.parent_get_value_method.invoke_args(object, [value_name]) + new_value = self.parent_get_value_method.invoke_args(object, [ + value_name]) if new_value != None: if cur_relation_desc.is_multi_valued(): - merge_result = self.merge_object_lists(existing_value, new_value) + merge_result = self.merge_object_lists( + existing_value, new_value) new_value = merge_result["result"] self.set_value_internal(value_name, new_value) @@ -905,10 +930,11 @@ def set_value( for i in range(len(value)): cur_value = value[i] if cur_value != None: - result &= self.add_node(cur_value, name, force_set, track_change) + result &= self.add_node( + cur_value, name, force_set, track_change) self.relation_states[name] = self.RELATION_STATE_INITIALIZED return result # default behaviour - return super().set_value(name, value, force_set, track_change) \ No newline at end of file + return super().set_value(name, value, force_set, track_change) diff --git a/digitalpy/core/domain/object_id.py b/digitalpy/core/domain/object_id.py index 864d5032..c674fc47 100644 --- a/digitalpy/core/domain/object_id.py +++ b/digitalpy/core/domain/object_id.py @@ -1,4 +1,5 @@ import re +from typing import Union from digitalpy.core.main.object_factory import ObjectFactory import uuid @@ -8,8 +9,8 @@ class ObjectId: DELIMITER = ":" __dummy_id_pattern = "DigitalPy[A-Za-z0-9]{32}" - __id_pattern = None - __delimiter_pattern = None + __id_pattern = ".*" + __delimiter_pattern = ":" _num_pk_keys = {} __null_oid = None __fq_type = None @@ -17,12 +18,12 @@ class ObjectId: def __init__(self, type, id=[], prefix=""): self.prefix = prefix # TODO, properly implement the type checking of the object via the persistence facade - #self.persistence_facade = ObjectFactory.get_instance("persistencefacade") - #self.__fq_type = ( + # self.persistence_facade = ObjectFactory.get_instance("persistencefacade") + # self.__fq_type = ( # lambda: self.persistence_facade.get_fully_qualified_type(type) # if type != "NULL" # else "NULL" - #)() + # )() self.__fq_type = type if not isinstance(id, list): @@ -30,21 +31,22 @@ def __init__(self, type, id=[], prefix=""): else: self.id = id - #self.num_pks = ObjectId.get_number_of_pks(type) + # self.num_pks = ObjectId.get_number_of_pks(type) - #while len(self.id) < self.num_pks: + # while len(self.id) < self.num_pks: # self.id.append(self.get_dummy_id()) self.str_val = self.__str__() def __str__(self): - oid_str = self.__fq_type + ObjectId.DELIMITER + ObjectId.DELIMITER.join(self.id) + oid_str = self.__fq_type + ObjectId.DELIMITER + \ + ObjectId.DELIMITER.join(self.id) if len(self.prefix.strip()) > 0: oid_str = self.prefix + ObjectId.DELIMITER + oid_str self.str_val = oid_str return self.str_val - def get_id(self): + def get_id(self) -> list[str]: return self.id def get_type(self): @@ -70,7 +72,8 @@ def NULL_OID(): def get_number_of_pks(type): if type not in ObjectId._num_pk_keys: num_pks = 1 - persistence_facade = ObjectFactory.get_instance("persistencefacade") + persistence_facade = ObjectFactory.get_instance( + "persistencefacade") if persistence_facade.is_known_type(type): mapper = persistence_facade.get_mapper(type) num_pks = len(mapper.get_pk_names) @@ -89,11 +92,10 @@ def is_valid(oid): return True @staticmethod - def parse(oid): + def parse(oid) -> Union['ObjectId', None]: if isinstance(oid, ObjectId): return oid - return None - # TODO: properly implement parsing oid string + oid_parts = ObjectId.parse_oid_string(oid) if not oid_parts: return None @@ -102,42 +104,50 @@ def parse(oid): ids = oid_parts["id"] prefix = oid_parts["prefix"] - if not ObjectFactory.get_instance("persistence_facade").is_known_type(type): - return None + # Sections commented out as the persistence facade is not yet implemented + # if not ObjectFactory.get_instance("persistence_facade").is_known_type(type): + # return None - num_pks = ObjectId.get_number_of_pks(type) - if num_pks == None or num_pks != len(ids): - return None + # num_pks = ObjectId.get_number_of_pks(type) + # if num_pks == None or num_pks != len(ids): + # return None return ObjectId(type, ids, prefix) @staticmethod def get_delimiter_pattern(): - if ObjectId.delimiter_pattern == None: - ObjectId.delimiter_pattern = "/" + ObjectId.DELIMITER + "/" - return ObjectId.delimiter_pattern + if ObjectId.__delimiter_pattern == None: + ObjectId.__delimiter_pattern = "\\"+ObjectId.DELIMITER + return ObjectId.__delimiter_pattern @staticmethod def parse_oid_string(oid: str): if len(oid) == 0: return None - oid_parts = re.split(ObjectId.get_delimiter_pattern(), oid) + + oid_parts = re.split(ObjectId.__delimiter_pattern, oid) if len(oid_parts) < 2: return None - if ObjectId.__id_pattern == None: - ObjectId.__id_pattern = "/^[0-9]*$|^" + ObjectId.__dummy_id_pattern + "$/" + # get the ids from the oid ids = [] next_part = oid_parts.pop() - while next_part != None and re.split(ObjectId.__id_pattern, next_part) == 1: - int_next_part = int(next_part) - if next_part == str(int_next_part): - ids.append(int_next_part) - else: + while next_part is not None and re.match(ObjectId.__id_pattern, next_part): + try: + ids.append(int(next_part)) + except ValueError: ids.append(next_part) + next_part = oid_parts.pop() if oid_parts else None ids.reverse() - type = next_part - prefix = ObjectId.DELIMITER + oid_parts + # get the type + type_ = ids.pop(0) + + # get the prefix + prefix = ObjectId.DELIMITER.join(oid_parts) - return {"type": type, "id": ids, "prefix": prefix} \ No newline at end of file + return { + 'type': type_, + 'id': ids, + 'prefix': prefix + } diff --git a/digitalpy/core/file/files_facade.py b/digitalpy/core/file/files_facade_dev.py similarity index 100% rename from digitalpy/core/file/files_facade.py rename to digitalpy/core/file/files_facade_dev.py diff --git a/digitalpy/core/health/domain/service_health_category.py b/digitalpy/core/health/domain/service_health_category.py new file mode 100644 index 00000000..ea73c6c4 --- /dev/null +++ b/digitalpy/core/health/domain/service_health_category.py @@ -0,0 +1,10 @@ +from enum import Enum + +class ServiceHealthCategory(Enum): + """Service Health Category""" + # the service is operational with all metrics in acceptable ranges + OPERATIONAL = "Operational" + # some or all metrics are outside of acceptable ranges + DEGRADED = "Degraded" + # the service is not responding to requests + UNRESPONSIVE = "Unresponsive" \ No newline at end of file diff --git a/digitalpy/core/health/health_facade.py b/digitalpy/core/health/health_facade_dev.py similarity index 100% rename from digitalpy/core/health/health_facade.py rename to digitalpy/core/health/health_facade_dev.py diff --git a/digitalpy/core/logic/logic_facade.py b/digitalpy/core/logic/logic_facade_dev.py similarity index 100% rename from digitalpy/core/logic/logic_facade.py rename to digitalpy/core/logic/logic_facade_dev.py diff --git a/digitalpy/core/main/DigitalPy.py b/digitalpy/core/main/DigitalPy.py index 711a76dd..16fe5781 100644 --- a/digitalpy/core/main/DigitalPy.py +++ b/digitalpy/core/main/DigitalPy.py @@ -1,32 +1,58 @@ ####################################################### -# +# # DigitalPy.py # Python implementation of the Class DigitalPy # Generated by Enterprise Architect # Created on: 28-Dec-2022 1:18:23 PM # Original author: FreeTAKTeam -# +# ####################################################### +from io import StringIO import multiprocessing import os import pathlib +import sys +from threading import Event +from time import sleep +from typing import Callable, TYPE_CHECKING +import signal from digitalpy.core.digipy_configuration.configuration import Configuration from digitalpy.core.digipy_configuration.impl.inifile_configuration import InifileConfiguration from digitalpy.core.component_management.impl.component_registration_handler import ComponentRegistrationHandler +from digitalpy.core.telemetry.tracer import Tracer +from digitalpy.core.telemetry.tracing_provider import TracingProvider +from digitalpy.core.zmanager.response import Response +from digitalpy.core.service_management.domain.service_manager_operations import ServiceManagerOperations + +from digitalpy.core.zmanager.subject import Subject +from digitalpy.core.zmanager.impl.zmq_pusher import ZMQPusher +from digitalpy.core.zmanager.impl.zmq_subscriber import ZmqSubscriber +from digitalpy.core.zmanager.request import Request from digitalpy.core.main.factory import Factory from digitalpy.core.main.impl.default_factory import DefaultFactory from digitalpy.core.main.object_factory import ObjectFactory +from digitalpy.core.service_management.controllers.service_management_main import ServiceManagementMain +from digitalpy.core.service_management.digitalpy_service import COMMAND_PROTOCOL, COMMAND_ACTION + +if TYPE_CHECKING: + from digitalpy.core.domain.domain.service_health import ServiceHealth + -class DigitalPy: +class DigitalPy(ZmqSubscriber, ZMQPusher): """this is the executable of the digitalPy framework, providing the starting point for a bare bone application. """ + + service_id = "digitalpy" + def __init__(self): # Set up necessary resources and configurations for the application to run self.resources = [] self.configuration: Configuration = InifileConfiguration("") + self.routing_proxy_service: Subject + # register the digitalpy action mapping under ../digitalpy/action_mapping.ini self.configuration.add_configuration( str( @@ -38,16 +64,47 @@ def __init__(self): ) # the central digitalpy configuration used throughout the application self.factory: Factory = DefaultFactory(self.configuration) - + # register the factory and configuration to the object factory singleton ObjectFactory.configure(self.factory) ObjectFactory.register_instance("configuration", self.configuration) - + # factory instance is registered for use by the routing worker so that # the instances in the instance dictionary can be preserved when the # new object factory is instantiated in the sub-process ObjectFactory.register_instance("factory", self.factory) + self.initialize_tracing() + + ZmqSubscriber.__init__(self, ObjectFactory.get_instance("formatter")) + ZMQPusher.__init__(self, ObjectFactory.get_instance("formatter")) + + self.responses: dict[str, Response] = {} + + def set_zmanager_address(self): + self.subject_address: str = self.configuration.get_value( + "subject_address", "Service") + self.subject_port: int = int( + self.configuration.get_value("subject_port", "Service")) + self.subject_protocol: str = self.configuration.get_value( + "subject_protocol", "Service") + self.integration_manager_address: str = self.configuration.get_value( + "integration_manager_address", "Service") + self.integration_manager_port: int = int( + self.configuration.get_value("integration_manager_port", "Service")) + self.integration_manager_protocol: str = self.configuration.get_value( + "integration_manager_protocol", "Service") + + def initialize_tracing(self): + # the central tracing provider + self._tracing_provider: TracingProvider = self.factory.get_instance( + "tracingprovider") + + self._tracing_provider.initialize_tracing() + + self.tracer: Tracer = self._tracing_provider.create_tracer( + "DigitalPy.Main") + def register_components(self): """register all components of the application """ @@ -64,31 +121,71 @@ def register_components(self): for digipy_component in digipy_components: ComponentRegistrationHandler.register_component( - digipy_component, + digipy_component, # type: ignore "digitalpy.core", - self.configuration, + self.configuration, # type: ignore ) - def start(self): + def test_event_loop(self): + """ the main event loop of the application should be called within a continuous while loop + """ + sleep(1) + + def event_loop(self): + """ the main event loop of the application should be called within a continuous while loop + """ + # TODO: what should this be in the default case + sleep(1) + + def start(self, testing: bool = False): # type: ignore """Begin the execution of the application, this should be overriden by any inheriting classes""" + self.register_components() self.configure() self.start_services() + if not testing: + while True: + try: + self.event_loop() + except Exception as ex: # pylint: disable=broad-except + self.handle_exception(ex) + # TODO: add a testing flag to the configuration + elif testing: + while True: + try: + self.teardown_connections() + except Exception as ex: # pylint: disable=broad-except + self.handle_exception(ex) + + def handle_exception(self, error: Exception): + """Deal with errors that occur during the execution of the application + """ + print("error thrown :" + str(error)) def start_services(self): - self.start_routing_proxy_service() + self.set_zmanager_address() self.start_integration_manager_service() + self.start_routing_proxy_service() + self.start_service_manager() + self.initialize_connections() + + def initialize_connections(self): + ZMQPusher.initiate_connections( + self, self.subject_port, self.subject_address, self.service_id) + self.broker_connect(self.integration_manager_address, self.integration_manager_port, + self.integration_manager_protocol, self.service_id, COMMAND_PROTOCOL) def start_routing_proxy_service(self): """this function is responsible for starting the routing proxy service""" try: # begin the routing proxy - self.routing_proxy_service = ObjectFactory.get_instance("Subject") - proc = multiprocessing.Process( + self.routing_proxy_service: Subject = ObjectFactory.get_instance( + "Subject") + self.routing_proxy_process = multiprocessing.Process( target=self.routing_proxy_service.begin_routing ) - proc.start() + self.routing_proxy_process.start() return 1 @@ -100,11 +197,11 @@ def stop_routing_proxy_service(self): try: # TODO: add a pre termination call to shutdown workers and sockets before a # termination to prevent hanging resources - if self.routing_proxy_service.is_alive(): - self.routing_proxy_service.terminate() - self.routing_proxy_service.join() + if self.routing_proxy_process.is_alive(): + self.routing_proxy_process.terminate() + self.routing_proxy_process.join() else: - self.routing_proxy_service.join() + self.routing_proxy_process.join() return 1 except Exception as e: return -1 @@ -118,9 +215,11 @@ def start_integration_manager_service(self) -> bool: try: # begin the integration_manager_service - self.integration_manager_service = ObjectFactory.get_instance("IntegrationManager") - proc = multiprocessing.Process(target=self.integration_manager_service.start) - proc.start() + self.integration_manager_service = ObjectFactory.get_instance( + "IntegrationManager") + self.integration_manager_process = multiprocessing.Process( + target=self.integration_manager_service.start) + self.integration_manager_process.start() return True except Exception as ex: @@ -133,18 +232,153 @@ def stop_integration_manager_service(self) -> bool: bool: True if successful, False otherwise. """ try: - if self.integration_manager_service.is_alive(): - self.integration_manager_service.terminate() - self.integration_manager_service.join() + if self.integration_manager_process.is_alive(): + self.integration_manager_process.terminate() + self.integration_manager_process.join() else: - self.routing_proxy_service.join() + self.integration_manager_process.join() + return True + except Exception as ex: + raise ex + + def start_service_manager(self) -> bool: + """Starts the service manager. + + Returns: + bool: True if successful, False otherwise. + """ + try: + + # begin the service_manager + self.service_manager: ServiceManagementMain = ObjectFactory.get_instance( + "ServiceManager") + self.service_manager_process = multiprocessing.Process(target=self.service_manager.start, args=(ObjectFactory.get_instance( + "factory"), ObjectFactory.get_instance("tracingprovider"), ComponentRegistrationHandler.component_index)) + self.service_manager_process.start() + return True + except Exception as ex: + raise ex + + def stop_service_manager(self) -> bool: + """Stops the service manager. + + Returns: + bool: True if successful, False otherwise. + """ + try: + + self.stop_service(self.configuration.get_value( + "service_id", "servicemanager")) + self.service_manager_process.join(10) + + if self.service_manager_process.is_alive(): + + self.service_manager_process.terminate() + self.service_manager_process.join() + else: + self.service_manager_process.join() + return True + except Exception as ex: + raise ex + + def get_all_service_health(self) -> dict[str, 'ServiceHealth']: + """Gets the health of all services from the service manager.""" + try: + req: Request = ObjectFactory.get_new_instance("Request") + req.set_action("GetAllServiceHealth") + req.set_context(self.configuration.get_value( + "service_id", "ServiceManager")) + req.set_value("command", "get_all_service_health") + req.set_format("pickled") + self.subject_send_request(req, COMMAND_PROTOCOL, self.configuration.get_value( + "service_id", "ServiceManager")) + response = self.broker_receive_response(req.get_id(), timeout=10) + if response is None: + raise IOError( + "No response received from the service manager in time") + return response.get_value("message") + except Exception as ex: + raise ex + + def start_service(self, service_id: str) -> bool: + """Starts a service. + + Args: + service_id (str): The unique id of the service to start. + + Returns: + bool: True if successful, False otherwise. + """ + try: + req: Request = ObjectFactory.get_new_instance("Request") + req.set_action("StartServer") + req.set_context(self.configuration.get_value( + "service_id", "ServiceManager")) + req.set_value( + "command", ServiceManagerOperations.START_SERVICE.value) + req.set_value("target_service_id", service_id) + req.set_format("pickled") + self.subject_send_request(req, COMMAND_PROTOCOL, self.configuration.get_value( + "service_id", "ServiceManager")) + return True + except Exception as ex: + raise ex + + def stop_service(self, service_id: str) -> bool: + """Starts a service. + + Args: + service_id (str): The unique id of the service to start. + + Returns: + bool: True if successful, False otherwise. + """ + try: + req: Request = ObjectFactory.get_new_instance("Request") + req.set_action(COMMAND_ACTION) + req.set_context(self.configuration.get_value( + "service_id", "servicemanager")) + req.set_value( + "command", ServiceManagerOperations.STOP_SERVICE.value) + req.set_value("target_service_id", service_id) + req.set_format("pickled") + self.subject_send_request(req, COMMAND_PROTOCOL, self.configuration.get_value( + "service_id", "ServiceManager")) + return True + except Exception as ex: + raise ex + + def restart_service(self, service_id: str) -> bool: + """Starts a service. + + Args: + service_id (str): The unique id of the service to start. + + Returns: + bool: True if successful, False otherwise. + """ + try: + req: Request = ObjectFactory.get_new_instance("Request") + req.set_action(COMMAND_ACTION) + req.set_context(self.configuration.get_value( + "service_id", "ServiceManager")) + req.set_value( + "command", ServiceManagerOperations.RESTART_SERVICE.value) + req.set_value("target_service_id", service_id) + req.set_format("pickled") + self.subject_send_request(req, COMMAND_PROTOCOL, self.configuration.get_value( + "service_id", "ServiceManager")) return True except Exception as ex: raise ex def stop(self): - # End the execution of the application - pass + """End the execution of the application""" + self.stop_service_manager() + self.stop_integration_manager_service() + self.stop_routing_proxy_service() + + raise SystemExit def restart(self): # End and then restart the execution of the application @@ -162,7 +396,6 @@ def load_state(self): def configure(self): """Set or modify the configuration of the application""" - def get_status(self): # Retrieve the current status of the application (e.g. running, stopped, etc.) pass @@ -171,11 +404,7 @@ def get_logs(self): # Retrieve the log records generated by the application pass - def handle_errors(self, error): - # Deal with errors that occur during the execution of the application - pass - def shutdown(self): # Close all resources and terminate the application self.resources = [] - self.configurations = {} \ No newline at end of file + self.configurations = {} diff --git a/digitalpy/core/main/controller.py b/digitalpy/core/main/controller.py index d2247ff0..5cd1bd2f 100644 --- a/digitalpy/core/main/controller.py +++ b/digitalpy/core/main/controller.py @@ -32,8 +32,8 @@ def __init__( action_mapper: ActionMapper, configuration: Configuration, ): - self.request = request - self.response = response + self.request: Request = request + self.response: Response = response self.action_mapper = action_mapper self.configuration = configuration @@ -73,10 +73,10 @@ def get_request(self): def get_response(self): return self.response - def execute_sub_action(self, action): + def execute_sub_action(self, action) -> Response: cur_request = self.get_request() cur_response = self.get_response() - sub_request = ObjectFactory.get_new_instance("request") + sub_request: Request = ObjectFactory.get_new_instance("request") sub_request.set_sender(self.__class__.__name__) sub_request.set_context(cur_request.get_context()) sub_request.set_action(action) diff --git a/digitalpy/core/main/factory.py b/digitalpy/core/main/factory.py index 95513593..503be601 100644 --- a/digitalpy/core/main/factory.py +++ b/digitalpy/core/main/factory.py @@ -1,10 +1,11 @@ from abc import abstractmethod, ABC +from typing import Any class Factory(ABC): @abstractmethod - def get_instance(self, name, dynamic_configuration={}) -> object: + def get_instance(self, name, dynamic_configuration={}) -> Any: """Get an instance from the configuration. Instances created with this method might be shared (depending on the __shared configuration property).""" @@ -26,4 +27,8 @@ def add_interfaces(self, interfaces: dict): @abstractmethod def clear(self): - """Delete all created instances""" \ No newline at end of file + """Delete all created instances""" + + @abstractmethod + def clear_instance(self, name): + """Delete a specific instance""" \ No newline at end of file diff --git a/digitalpy/core/main/impl/default_factory.py b/digitalpy/core/main/impl/default_factory.py index 11938d27..04eeab93 100644 --- a/digitalpy/core/main/impl/default_factory.py +++ b/digitalpy/core/main/impl/default_factory.py @@ -105,17 +105,22 @@ def create_instance(self, name, configuration, instance_key): ): continue param_instance_key = param_name.lower().replace("_", "") + # first check the configuration section for the parameter if param_name in configuration: c_params[param_name] = self.resolve_value( configuration[param_name] ) + # then check if a parameter has already been initialized elif param_instance_key in self.instances: c_params[param_name] = self.instances[param_instance_key] + # check if a section with the name of the parameter exists elif self.configuration.has_section(param_name): c_params[param_name] = self.get_instance(param_name) + # check if a section with the name of the parameter in lowercase exists elif self.configuration.has_section(param_instance_key): - c_params[param_name] = self.get_instance(param_instance_key) - elif param_default == None: + c_params[param_name] = self.get_instance( + param_instance_key) + elif isinstance(param_default, inspect._empty): raise Exception( f"constructor parameter {param_name} in class {name} cannot be injected" ) @@ -223,3 +228,8 @@ def get_new_instance(self, name, dynamic_configuration={}): configuration = {**dynamic_configuration, "__shared": False} instance = self.get_instance(name, configuration) return instance + + def clear_instance(self, name): + instance_key = name.lower() + if instance_key in self.instances: + del self.instances[instance_key] diff --git a/digitalpy/core/main/object_factory.py b/digitalpy/core/main/object_factory.py index 771a599e..c5bf1ee0 100644 --- a/digitalpy/core/main/object_factory.py +++ b/digitalpy/core/main/object_factory.py @@ -1,5 +1,5 @@ from digitalpy.core.main.factory import Factory - +from typing import Any class ObjectFactory: __factory = None @@ -14,14 +14,14 @@ def is_configured(): return ObjectFactory.__factory != None @staticmethod - def get_instance(name, dynamic_configuration={}) -> object: + def get_instance(name, dynamic_configuration={}) -> Any: """Get an instance from the configuration. Instances created with this method might be shared (depending on the __shared configuration property).""" ObjectFactory.__check_config() return ObjectFactory.__factory.get_instance(name, dynamic_configuration) @staticmethod - def get_new_instance(name, dynamic_configuration={}): + def get_new_instance(name, dynamic_configuration={}) -> Any: """Get a new instance from the configuration. Instances created with this method are not shared.""" ObjectFactory.__check_config() return ObjectFactory.__factory.get_new_instance(name, dynamic_configuration) diff --git a/digitalpy/core/network/domain/Network.py b/digitalpy/core/network/domain/Network.py deleted file mode 100644 index 90c64213..00000000 --- a/digitalpy/core/network/domain/Network.py +++ /dev/null @@ -1,48 +0,0 @@ -####################################################### -# -# Network.py -# Python implementation of the Class Network -# Generated by Enterprise Architect -# Created on: 28-Dec-2022 10:14:16 AM -# Original author: Giu Platania -# -####################################################### - - -import sqlalchemy -from sqlalchemy import create_engine, Column, Integer, String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker - -Base = declarative_base() - -class Network(Base): - __tablename__ = 'network_settings' - id = Column(Integer, primary_key=True) - ip_address = Column(String) - subnet_mask = Column(String) - default_gateway = Column(String) - dns_servers = Column(String) - -def configure_network_settings(ip_address=None, subnet_mask=None, default_gateway=None, dns_servers=None): - engine = create_engine('sqlite:///network_settings.db') - Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - session = Session() - - network_settings = session.query(NetworkSettings).first() - if not network_settings: - network_settings = NetworkSettings() - session.add(network_settings) - - if ip_address: - network_settings.ip_address = ip_address - if subnet_mask: - network_settings.subnet_mask = subnet_mask - if default_gateway: - network_settings.default_gateway = default_gateway - if dns_servers: - network_settings.dns_servers = dns_servers - - session.commit() - session.close() \ No newline at end of file diff --git a/digitalpy/core/network/domain/Networkproperty.py b/digitalpy/core/network/domain/Networkproperty.py deleted file mode 100644 index 823c711b..00000000 --- a/digitalpy/core/network/domain/Networkproperty.py +++ /dev/null @@ -1,24 +0,0 @@ -####################################################### -# -# componentproperty.py -# Python implementation of the Class componentproperty -# Generated by Enterprise Architect -# Created on: 28-Dec-2022 10:14:14 AM -# Original author: Giu Platania -# -####################################################### - - -class componentproperty(CoTNode): -# default constructor def __init__(self): - - def __init__(): - pass - - @CoTProperty - def componentpropertyattribute(): - pass - - @componentpropertyattribute.setter - def componentpropertyattribute( = None): - pass \ No newline at end of file diff --git a/digitalpy/core/network/domain/client_status.py b/digitalpy/core/network/domain/client_status.py new file mode 100644 index 00000000..3b9668b8 --- /dev/null +++ b/digitalpy/core/network/domain/client_status.py @@ -0,0 +1,9 @@ + +from enum import Enum + +class ClientStatus(Enum): + CONNECTING = "CONNECTING" + CONNECTED = "CONNECTED" + DISCONNECTED = "DISCONNECTED" + TERMINATED = "TERMINATED" + diff --git a/digitalpy/core/network/domain/model_constants/NetworkPropertyVariables.py b/digitalpy/core/network/domain/model_constants/NetworkPropertyVariables.py deleted file mode 100644 index ff4d37ce..00000000 --- a/digitalpy/core/network/domain/model_constants/NetworkPropertyVariables.py +++ /dev/null @@ -1,19 +0,0 @@ -####################################################### -# -# ComponentPropertyVariables.py -# Python implementation of the Class ComponentPropertyVariables -# Generated by Enterprise Architect -# Created on: 28-Dec-2022 10:14:15 AM -# Original author: Giu Platania -# -####################################################### - - -class ComponentPropertyVariables: -# default constructor def __init__(self): - - %WRAP_COMMENT(linkAttNotes, genOptWrapComment, """"", "# ")% - self.COMPONENTPROPERTYATTRIBUTE = COMPONENTPROPERTYATTRIBUTE - - def __init__(): - pass \ No newline at end of file diff --git a/digitalpy/core/network/domain/model_constants/__init__.py b/digitalpy/core/network/domain/model_constants/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/digitalpy/core/network/domain/model_constants/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/digitalpy/core/network/domain/network_client.py b/digitalpy/core/network/domain/network_client.py new file mode 100644 index 00000000..87bf505d --- /dev/null +++ b/digitalpy/core/network/domain/network_client.py @@ -0,0 +1,101 @@ +from digitalpy.core.domain.node import Node +from digitalpy.core.parsing.load_configuration import ModelConfiguration + +from .client_status import ClientStatus + + +class NetworkClient(Node): + """this class represents a network client in the digitalpy framework, + it is the base network client implementation and can be expanded""" + + def __init__(self, configuration=None, model=None, oid=None, node_type="network_client") -> None: + super().__init__(node_type, configuration=configuration, + model=model, oid=oid) # type: ignore + # the id of the client + self._id: bytes + # the status of the client + self._status: ClientStatus = ClientStatus.CONNECTING + # id of the related service + self._service_id: str + # the protocol used by the client + self._protocol: str + + @property + def protocol(self) -> str: + """get the protocol of the client + + Returns: + str: the protocol of the client + """ + return self._protocol + + @protocol.setter + def protocol(self, protocol: str): + """set the protocol of the client + + Args: + protocol (str): the protocol of the client + """ + if not isinstance(protocol, str): + raise TypeError("'protocol' must be an instance of str") + self._protocol = protocol + + @property + def service_id(self) -> str: + """get the service id of the client + + Returns: + str: the service id of the client + """ + return self._service_id + + @service_id.setter + def service_id(self, service_id: str): + """set the service id of the client + + Args: + service_id (str): the service id of the client + """ + if not isinstance(service_id, str): + raise TypeError("'service_id' must be an instance of str") + self._service_id = service_id + + @property + def status(self) -> ClientStatus: + """get the status of the client + + Returns: + str: the status of the client + """ + return self._status + + @status.setter + def status(self, status: ClientStatus): + """set the status of the client + + Args: + status (str): the status of the client + """ + if status not in ClientStatus: + raise ValueError("'status' must be a valid ClientStatus") + self._status = status + + @property + def id(self) -> bytes: + """get the id of the client + + Returns: + bytes: the id of the client + """ + return self._id + + @id.setter + def id(self, id: bytes): + """set the id of the client + + Args: + id (bytes): the id of the client + """ + if not isinstance(id, bytes): + raise TypeError("'id' must be an instance of int") + self._id = id diff --git a/digitalpy/core/network/impl/network_tcp.py b/digitalpy/core/network/impl/network_tcp.py new file mode 100644 index 00000000..09072a52 --- /dev/null +++ b/digitalpy/core/network/impl/network_tcp.py @@ -0,0 +1,165 @@ +"""This file defines a class `TCPNetwork` that implements the `NetworkAsyncInterface`. This class is used to establish a network communication using the TCP protocol. It uses the ZeroMQ library for creating the socket and context for communication. + +The class has methods for initializing the network (`intialize_network`) and connecting a client to the server (`connect_client`). The `initialize_network` method sets up the ZeroMQ context and socket, binds it to the provided host and port. The `connect_client` method is used to connect a client to the server using the client's identity. + +The class also maintains a dictionary of clients connected to the network, with their identities as keys. +""" + +from typing import Dict, List, TYPE_CHECKING, Union +import zmq +from digitalpy.core.main.object_factory import ObjectFactory +from digitalpy.core.network.domain.client_status import ClientStatus +from digitalpy.core.domain.object_id import ObjectId + +from digitalpy.core.network.network_async_interface import NetworkAsyncInterface +from digitalpy.core.domain.domain.network_client import NetworkClient +from digitalpy.core.zmanager.request import Request +from digitalpy.core.zmanager.response import Response + +if TYPE_CHECKING: + from digitalpy.core.domain.domain_facade import Domain + + +class TCPNetwork(NetworkAsyncInterface): + """this class implements the NetworkAsyncInterface using the TCP protocol realizing + the simplified approach to network communication + """ + + def __init__(self): + self.host: str = None # type: ignore + self.port: int = None # type: ignore + self.socket: zmq.Socket = None # type: ignore + self.context: zmq.Context = None # type: ignore + self.clients: Dict[str, NetworkClient] = {} + + def intialize_network(self, host: str, port: int): + self.context = zmq.Context() + self.socket = self.context.socket(zmq.STREAM) + self.socket.getsockopt_string(zmq.IDENTITY) + self.socket.bind(f"tcp://{host}:{port}") + + def connect_client(self, identity: bytes) -> NetworkClient: + """connect a client to the server + + Args: + identity (bytes): the identity of the client + """ + client = self.handle_client_connection(identity) + + self.clients[str(client.id)] = client + + return client + + def handle_client_disconnection(self, client: NetworkClient): + """disconnect a client from the server + + Args: + identity (NetworkClient): the network client to disconnect + """ + client.status = ClientStatus.DISCONNECTED + + return self.clients.pop(str(client.id)) + + def handle_empty_msg(self, identity: bytes, request: Request) -> NetworkClient: + """handle an empty message from the client + + Args: + client (NetworkClient): the client that sent the empty message + """ + client = self.clients.get(str(identity)) + if not client: + request.set_value("action", "connection") + return self.connect_client(identity) + else: + request.set_value("action", "disconnection") + return self.handle_client_disconnection(client) + + def service_connections(self, max_requests=1000) -> List[Request]: + """service all connections to the server and return a list of requests + + Returns: + List[request]: a list of requests from the clients + """ + requests = [] + try: + + # avoid blocking indefinitely + while len(requests) < max_requests: + + # receive the identity of the client and it's message contents + identity = self.receive_message() + msg = self.receive_message() + + # construct a request + req: Request = ObjectFactory.get_new_instance("Request") + req.set_value("data", msg) + + if msg == b'': + client = self.handle_empty_msg(identity, req) + + else: + client = self.clients.get(str(identity)) + + req.set_value("client", client) + requests.append(req) + + # receive messages should throw an exception when there are no messages to receive + except zmq.Again: + pass + + return requests + + def teardown_network(self): + if self.socket: + self.socket.close() + + def receive_message(self, blocking: bool = False) -> bytes: + # Implement logic to receive a message from any client + if not blocking: + msg = self.socket.recv(zmq.NOBLOCK) + else: + msg = self.socket.recv() + return msg + + def handle_client_connection(self, network_id: bytes) -> NetworkClient: + """handle a client connection + """ + oid = ObjectId("network_client", id=str(network_id)) + client: NetworkClient = ObjectFactory.get_new_instance( + "DefaultClient", dynamic_configuration={"oid": oid}) + client.id = network_id + client.status = ClientStatus.CONNECTED + return client + + def receive_message_from_client(self, client: NetworkClient, blocking: bool = False) -> Request: + # Implement logic to receive a message from a specific client + return None # type: ignore + + def send_message_to_client(self, message: Response, client: NetworkClient): + try: + for message_data in message.get_value("message"): + self.socket.send_multipart( + [client.id, message_data]) + except Exception as e: + raise IOError( + f"Failed to send message to client {client}: {str(e)}") from e + + def send_message_to_clients(self, message: Response, clients: Union[List[NetworkClient], List[str]]): + for client in clients: + if isinstance(client, str): + oid = ObjectId.parse(client) + if oid is None: + raise ValueError(f"Invalid client id: {client}") + client = self.clients.get(oid.get_id()[0]) + if client is None: + raise ValueError(f"Client not found: {client}") + self.send_message_to_client(message, client) + + def send_message_to_all_clients(self, message: Response, suppress_failed_sending: bool = False): + for client in self.clients.values(): + try: + self.send_message_to_client(message, client) + except IOError as e: + if not suppress_failed_sending: + raise IOError( + f"Failed to send message to client {client}: {str(e)}") diff --git a/digitalpy/core/network/network_async_interface.py b/digitalpy/core/network/network_async_interface.py new file mode 100644 index 00000000..f00fcc78 --- /dev/null +++ b/digitalpy/core/network/network_async_interface.py @@ -0,0 +1,67 @@ +from abc import abstractmethod +from typing import List +from digitalpy.core.domain.domain.network_client import NetworkClient +from digitalpy.core.network.network_interface import NetworkInterface +from digitalpy.core.zmanager.request import Request + + +class NetworkAsyncInterface(NetworkInterface): + """Network Async Interface class. Defines the interface for implementations of asynchronous networking + """ + + @abstractmethod + def service_connections(self) -> List[Request]: + """service all connections to the server and return a list of Requests + + Returns: + List[Request]: _description_ + """ + + @abstractmethod + def intialize_network(self, host: str, port: int): + """initialize the network connection, bind to the port and host. + """ + + @abstractmethod + def teardown_network(self): + """stop listening for messages from the network and release all files and resources + """ + + @abstractmethod + def receive_message(self, blocking: bool = False) -> Request: + """receive the next queued message from the network + Args: + blocking (bool, optional): whether or not to block until a message is received. Defaults to False. + Returns: + Request: the received message + """ + + @abstractmethod + def receive_message_from_client(self, client: NetworkClient, blocking: bool = False) -> Request: + """receive the next queued message from the network from a specific client + Args: + client_id (int): the id of the client to receive the message from + blocking (bool, optional): whether or not to block until a message is received. Defaults to False. + """ + + @abstractmethod + def send_message_to_client(self, message: Request, client: NetworkClient): + """send a message to the network + Args: + message (Request): the message to send + client_id (int): the id of the client to send the message to + + Raises: + ValueError: if the client_id is not valid + IOError: if the message cannot be sent + """ + + @abstractmethod + def send_message_to_all_clients(self, message: Request, suppress_failed_sending: bool = False): + """ send a message to all clients on the network + Args: + message (Request): the message to send + suppress_failed_sending (bool, optional): whether or not to suppress any errors that occur when sending the message to a client. Defaults to False. + Raises: + IOError: if the message cannot be sent to one or all clients + """ diff --git a/digitalpy/core/network/network_interface.py b/digitalpy/core/network/network_interface.py new file mode 100644 index 00000000..a6f7d625 --- /dev/null +++ b/digitalpy/core/network/network_interface.py @@ -0,0 +1,78 @@ +from abc import ABC, abstractmethod +from typing import List, Union +from digitalpy.core.main.object_factory import ObjectFactory +from digitalpy.core.domain.domain.network_client import NetworkClient +from digitalpy.core.zmanager.request import Request +from digitalpy.core.zmanager.response import Response + + +class NetworkInterface(ABC): + """Network Interface class. Defines the interface for all networking implementations + """ + + @abstractmethod + def handle_client_connection(self, network_id) -> NetworkClient: + """handle a client connection + """ + + @abstractmethod + def handle_client_disconnection(self, client: NetworkClient): + """handle a client disconnection + """ + + @abstractmethod + def service_connections(self) -> List[Request]: + """service all connections to the server and return a list of Requests + + Returns: + List[Request]: _description_ + """ + + @abstractmethod + def intialize_network(self, host: str, port: int): + """initialize the network connection, bind to the port and host. + """ + + @abstractmethod + def teardown_network(self): + """stop listening for messages from the network and release all files and resources + """ + + @abstractmethod + def receive_message(self, blocking: bool = False) -> Request: + """receive the next queued message from the network + Args: + blocking (bool, optional): whether or not to block until a message is received. Defaults to False. + Returns: + Request: the received message + """ + + @abstractmethod + def receive_message_from_client(self, client: NetworkClient, blocking: bool = False) -> Request: + """receive the next queued message from the network from a specific client + Args: + client_id (int): the id of the client to receive the message from + blocking (bool, optional): whether or not to block until a message is received. Defaults to False. + """ + + @abstractmethod + def send_message_to_clients(self, message: Response, clients: Union[List[NetworkClient], List[str]]): + """send a message to the network + Args: + message (Request): the message to send + client (NetworkClient): the network client to send the message to + + Raises: + ValueError: if the client_id is not valid + IOError: if the message cannot be sent + """ + + @abstractmethod + def send_message_to_all_clients(self, message: Response, suppress_failed_sending: bool = False): + """ send a message to all clients on the network + Args: + message (Request): the message to send + suppress_failed_sending (bool, optional): whether or not to suppress any errors that occur when sending the message to a client. Defaults to False. + Raises: + IOError: if the message cannot be sent to one or all clients + """ diff --git a/digitalpy/core/parsing/load_configuration.py b/digitalpy/core/parsing/load_configuration.py index a97cb217..2305a167 100644 --- a/digitalpy/core/parsing/load_configuration.py +++ b/digitalpy/core/parsing/load_configuration.py @@ -5,29 +5,35 @@ import json from typing import Dict + @dataclass class Relationship: min_occurs: int = 0 max_occurs: int = 1 + @dataclass class ConfigurationEntry: relationships: Dict[str, Relationship] = field(default_factory=dict) + @dataclass -class Configuration: +class ModelConfiguration: elements: Dict[str, ConfigurationEntry] = field(default_factory=dict) + class LoadConfiguration: def __init__(self, configuration_path_template: Template): self.configuration_path_template = configuration_path_template def find_configuration(self, message_type): message_type = message_type.lower() - message_configuration_path = self.configuration_path_template.substitute(message_type=message_type) + message_configuration_path = self.configuration_path_template.substitute( + message_type=message_type) if not os.path.exists(message_configuration_path): - raise Exception("configuration for %s not found" % message_configuration_path) - + raise Exception("configuration for %s not found" % + message_configuration_path) + # TODO: extend with more configuration formats if message_configuration_path.endswith(".json"): return self.parse_json_configuration(message_configuration_path) @@ -37,18 +43,20 @@ def find_configuration(self, message_type): def parse_json_configuration(self, message_configuration_path): with open(message_configuration_path, 'rb') as configuration_file: config = json.load(configuration_file)["definitions"] - configuration = Configuration() + configuration = ModelConfiguration() for class_name, class_values in config.items(): config_entry = ConfigurationEntry() for child_name, value in class_values.get("properties", {}).items(): if "$ref" in value: child_name = value["$ref"].split("/")[-1] config[child_name] - config_entry.relationships[child_name] = Relationship(value.get("minItems", 0), value.get("maxItems", 1)) + config_entry.relationships[child_name] = Relationship( + value.get("minItems", 0), value.get("maxItems", 1)) elif value["type"] == "array": if "$ref" in value["items"]: - child_name = value["items"]["$ref"].split("/")[-1] + child_name = value["items"]["$ref"].split("/")[-1] config[child_name] - config_entry.relationships[child_name] = Relationship(value.get("minItems", 0), value.get("maxItems", "*")) + config_entry.relationships[child_name] = Relationship( + value.get("minItems", 0), value.get("maxItems", "*")) configuration.elements[class_name] = config_entry - return configuration \ No newline at end of file + return configuration diff --git a/digitalpy/core/persistence/impl/default_persistent_object.py b/digitalpy/core/persistence/impl/default_persistent_object.py index 0f4f21fd..dee5b6bd 100644 --- a/digitalpy/core/persistence/impl/default_persistent_object.py +++ b/digitalpy/core/persistence/impl/default_persistent_object.py @@ -32,7 +32,8 @@ def __init__(self, oid: ObjectId = None, initial_data=None): data = {} self.attribute_descriptions = self.get_mapper().get_attributes() for cur_attribute_desc in self.attribute_descriptions: - data[cur_attribute_desc.get_name()] = cur_attribute_desc.get_default_value() + data[cur_attribute_desc.get_name( + )] = cur_attribute_desc.get_default_value() if initial_data is not None: data = {**data, **initial_data} @@ -45,7 +46,8 @@ def __init__(self, oid: ObjectId = None, initial_data=None): self._set_oid_internal(oid, False) def __initialize_mapper(self): - self.persistence_facade = ObjectFactory.get_instance("persistencefacade") + self.persistence_facade = ObjectFactory.get_instance( + "persistencefacade") if self.persistence_facade.is_known_type(self.__type): self.mapper = self.persistence_facade.get_mapper(self.__type) else: @@ -69,7 +71,8 @@ def set_property(self, name, value): old_value = self.get_property(name) self._properties[name] = value ObjectFactory.get_instance("eventManager").dispatch( - PropertyChangeEvent.NAME, PropertyChangeEvent(self, name, old_value, value) + PropertyChangeEvent.NAME, PropertyChangeEvent( + self, name, old_value, value) ) def get_properties(self): @@ -108,7 +111,8 @@ def set_value(self, name, value, force_set=False, track_change=True): if name in self.get_mapper().get_pk_names(): self.update_oid() if track_change: - DefaultPersistentObject.set_state(DefaultPersistentObject.STATE_DIRTY) + DefaultPersistentObject.set_state( + DefaultPersistentObject.STATE_DIRTY) self.changed_attributes[name] = True ObjectFactory.get_instance("eventManager").dispatch( ValueChangeEvent.NAME, @@ -135,4 +139,4 @@ def get_type(self) -> str: Returns: str: class name """ - return self.__class__.__name__ \ No newline at end of file + return self.__class__.__name__ diff --git a/digitalpy/core/queries/queries_facade.py b/digitalpy/core/queries/queries_facade_dev.py similarity index 100% rename from digitalpy/core/queries/queries_facade.py rename to digitalpy/core/queries/queries_facade_dev.py diff --git a/digitalpy/core/serialization/controllers/xml_serialization_controller.py b/digitalpy/core/serialization/controllers/xml_serialization_controller.py index 56328638..d77071eb 100644 --- a/digitalpy/core/serialization/controllers/xml_serialization_controller.py +++ b/digitalpy/core/serialization/controllers/xml_serialization_controller.py @@ -8,22 +8,15 @@ from lxml import etree import xmltodict + class XMLSerializationController(Controller): def __init__(self, request, response, action_mapper, configuration) -> None: super().__init__(request, response, action_mapper, configuration) - + def execute(self, method=None): getattr(self, method)(**self.request.get_values()) return self.response - def convert_xml_to_dict(self, message: str, **kwargs): - """converts the provided xml string to a dictionary - - Args: - message (str): xml string to be converted to a dictionary - """ - self.response.set_value("dict", xmltodict.parse(message)) - def serialize_node(self, node, **kwargs): """converts the provided node to an xml string @@ -47,7 +40,11 @@ def _serialize_node( Union[str, Element]: the original call to this method returns a string representing the xml the Element is only returned in the case of recursive calls """ - xml = node.xml + if node.extended: + xml = etree.fromstring(xmltodict.unparse(node.extended)) + else: + xml = etree.Element(tag_name) + # handles text data within tag if hasattr(node, "text"): xml.text = str(node.text) @@ -56,7 +53,8 @@ def _serialize_node( # below line is required because get_all_properties function returns only cot property names value = getattr(node, attribName) if hasattr(value, "__dict__"): - tagElement = self._serialize_node(value, attribName, level=level + 1) + tagElement = self._serialize_node( + value, attribName, level=level + 1) # TODO: modify so double underscores are handled differently try: if attribName[0] == "_": diff --git a/digitalpy/core/service_management/__init__.py b/digitalpy/core/service_management/__init__.py index c5bd8baf..f6891968 100644 --- a/digitalpy/core/service_management/__init__.py +++ b/digitalpy/core/service_management/__init__.py @@ -1 +1,40 @@ -"""similarly to component Management, this core function provides the ability to install, deinstall, discovery, start and stop services""" +""" +The Service Management component in DigitalPy is a core function designed to +manage the lifecycle and operations of various services within the framework. +It provides abstract capabilities for installing, deinstalling, discovering, +starting, and stopping services, aligning with the principles set in the +Network component for network type and communication protocols. + +* Lifecycle Management: + Installation and Deinstallation: Allows for the installation and removal + of services within the DigitalPy environment. + +* Service Discovery: + Facilitates the discovery of available services, aiding + in dynamic service management and integration. + +* Start/Stop Mechanisms: + Provides the ability to start and stop services + dynamically, ensuring flexibility and responsiveness in resource + management. + +* Service Isolation and Association: + Ensures each service runs in a thread and is isolated from others, + Associates each service with a specific port and network type, as defined + in the Network component. + Supports various data formats and protocols such as XML, JSON, Protobuf, + etc. + +* Integration with ZManager: + Implements a Subscriber pattern of the ZManager, allowing services to + subscribe to specific topics published by the Integration Manager. + +* Message Handling: + Implements a push pattern to forward received messages to the Subject + (Ventilator). + +* Interoperability and Standardization: Ensures compatibility with different + network types and communication protocols, as defined in the Network + component. Aligns with standardized practices for service management, ensuring a + consistent and efficient operational environment. +""" diff --git a/digitalpy/core/service_management/configuration/service_management_constants.py b/digitalpy/core/service_management/configuration/service_management_constants.py index 346bd01d..f5318be8 100644 --- a/digitalpy/core/service_management/configuration/service_management_constants.py +++ b/digitalpy/core/service_management/configuration/service_management_constants.py @@ -2,8 +2,6 @@ import pathlib from string import Template -COMPONENT_NAME = "ServiceManagement" - CONFIGURATION_FORMAT = "json" CURRENT_COMPONENT_PATH = pathlib.Path(__file__).parent.parent.absolute() @@ -50,3 +48,6 @@ MANIFEST_PATH = str( pathlib.PurePath(CURRENT_COMPONENT_PATH, "configuration/manifest.ini") ) + +# time to wait until a service is manually terminated +SERVICE_WAIT_TIME = 10 \ No newline at end of file diff --git a/digitalpy/core/service_management/controllers/service_management_main.py b/digitalpy/core/service_management/controllers/service_management_main.py new file mode 100644 index 00000000..7a790d09 --- /dev/null +++ b/digitalpy/core/service_management/controllers/service_management_main.py @@ -0,0 +1,392 @@ +""" +The Service Management component in DigitalPy is a core function designed +to manage the lifecycle and operations of various services within the framework. +It provides abstract capabilities for installing, deinstalling, discovering, starting, +and stopping services, aligning with the principles set in the Network component for network +type and communication protocols. + +The ServiceManager Service manages the following responsibilities: +* Lifecycle Management: + Installation and Deinstallation: Allows for the installation and removal + of services within the DigitalPy environment. + +* Service Discovery: Facilitates the discovery of available services, aiding + in dynamic service management and integration. + Start/Stop Mechanisms: Provides the ability to start and stop services + dynamically, ensuring flexibility and responsiveness in resource + management. + +* Service Isolation and Association: + Ensures each service runs in a thread and is isolated from others, + Associates each service with a specific port and network type, as defined + in the Network component. + Supports various data formats and protocols such as XML, JSON, Protobuf, + etc. +""" + +from datetime import datetime +from typing import Any, Dict, List + +from digitalpy.core.main.impl.default_factory import DefaultFactory +from digitalpy.core.service_management.domain.service_manager_operations \ + import ServiceManagerOperations +from digitalpy.core.service_management.domain.service_operations import ServiceOperations +from digitalpy.core.zmanager.request import Request +from digitalpy.core.main.object_factory import ObjectFactory +from digitalpy.core.service_management.domain.service_status import ServiceStatus +from digitalpy.core.telemetry.tracing_provider import TracingProvider +from digitalpy.core.service_management.controllers.service_management_process_controller \ + import ServiceManagementProcessController +from digitalpy.core.service_management.digitalpy_service \ + import DigitalPyService, COMMAND_PROTOCOL, COMMAND_ACTION +from digitalpy.core.parsing.formatter import Formatter +from digitalpy.core.service_management.domain.service_description import ServiceDescription +from digitalpy.core.digipy_configuration.configuration import Configuration +from digitalpy.core.zmanager.response import Response +from digitalpy.core.domain.domain.service_health import ServiceHealth +from digitalpy.core.health.domain.service_health_category import ServiceHealthCategory + + +class ServiceManagementMain(DigitalPyService): + """The Service Management component in DigitalPy is a core function designed + to manage the lifecycle and operations of various services within the framework. + It provides abstract capabilities for installing, deinstalling, discovering, starting, + and stopping services, aligning with the principles set in the Network component for network + type and communication protocols.""" + + def __init__( + self, + service_id: str, + subject_address: str, + subject_port: int, + subject_protocol: str, + integration_manager_address: str, + integration_manager_port: int, + integration_manager_protocol: str, + formatter: Formatter, + ): + """constructor for the ServiceManagementMain class + + Args: + subject_address (str): the address of the zmanager "subject" + subject_port (int): the port of the zmanager "subject" + integration_manager_address (str): the address of the zmanager "integration_manager" + integration_manager_port (int): the port of the zmanager "integration_manager" + formatter (Formatter): _description_ + """ + super().__init__( + service_id, + subject_address, + subject_port, + subject_protocol, + integration_manager_address, + integration_manager_port, + integration_manager_protocol, + formatter, + protocol=COMMAND_PROTOCOL, + network=None # type: ignore + ) + + # the service index is used to keep track of all registered services, + # their states, and their configurations + self._service_index: Dict[str, ServiceDescription] = {} + self.component_index: Dict[str, Configuration] = {} + self.process_controller: ServiceManagementProcessController + + def initialize_controllers(self): + """ + Initializes the controllers for service management. + """ + self.process_controller: ServiceManagementProcessController = ObjectFactory.get_instance( + "ServiceManagerProcessController") + + def stop(self): + """ + This method is used to stop the service manager. + """ + self.stop_all_services() + raise SystemExit + + def stop_all_services(self): + """This method is used to stop all services managed by the service manager""" + for service_id in self._service_index: + self.stop_service(service_id) + + def start(self, object_factory: DefaultFactory, tracing_provider: TracingProvider, + component_index: Dict[str, Configuration]): + """This method is used to start the service + + Args: + object_factory (DefaultFactory): + The object factory used for dependency injection. + tracing_provider (TracingProvider): + The tracing provider used for logging and monitoring. + component_index (Dict[str, Configuration]): + The index of components and their configurations. + + Returns: + None + """ + self.initialize_connections(COMMAND_PROTOCOL) + self.component_index = component_index + object_factory.clear_instance("servicemanager") + + ObjectFactory.configure(object_factory) + self.tracer = tracing_provider.create_tracer(self.service_id) + self.initialize_controllers() + self.initialize_connections(self.protocol) + + self.install_all_services() + + self.status = ServiceStatus.RUNNING + # member exists in parent class + self.execute_main_loop() # pylint: disable=no-member + + def event_loop(self): + """This method is used to run the service""" + commands = self.listen_for_commands() + + for command in commands: + self.handle_command(command) + + def install_all_services(self): + """ + Discover and install all services defined in the component manifest. + + This method iterates over the component index and retrieves the configuration + for each component. It then discovers services from the component + manifest. Finally it starts the services that have a default status of "RUNNING". + + Returns: + None + """ + for component_name in self.component_index: + component_configuration = self.component_index[component_name] + + # TODO: The service definitions should be moved out of the component manifest + # for better organization and separation of concerns. + for service_name in component_configuration.get_sections(): + if not service_name.endswith("Service"): + continue + + service_config = component_configuration.get_section( + service_name) + if service_config.get("default_status", ServiceStatus.STOPPED) \ + == ServiceStatus.RUNNING.value: + service_id = str(service_config.get("service_id")) + self.start_service(service_id) + + def handle_exception(self, exception: Exception): + """This method is used to handle an exception""" + # TODO: add logic to handle exceptions + print(exception) + + def handle_command(self, command: Response): + """This method is used to handle a command, commands are typically sent from the DigitalPy + application core through the ZManager""" + + if command.get_value("command") == str(ServiceManagerOperations.START_SERVICE.value): + self.start_service(command.get_value("target_service_id")) + + elif command.get_value("command") == str(ServiceManagerOperations.STOP_SERVICE.value): + self.stop_service(command.get_value("target_service_id")) + + elif command.get_value("command") == str(ServiceManagerOperations.RESTART_SERVICE.value): + self.stop_service(command.get_value("target_service_id")) + self.start_service(command.get_value("target_service_id")) + + elif command.get_value("command") == str(ServiceManagerOperations.GET_ALL_SERVICE_HEALTH.value): + all_service_health = self.get_all_service_health() + self.send_response_to_core(all_service_health, command.get_id()) + + def send_response_to_core(self, message: Any, command_id: str): + """This method is used to send a response to the DigitalPy application core""" + from digitalpy.core.main.DigitalPy import DigitalPy # pylint: disable=import-outside-toplevel + resp = ObjectFactory.get_new_instance("Response") + resp.set_value("message", message) + resp.set_action("publish") + resp.set_format("pickled") + resp.set_id(command_id) + self.subject_send_request( + resp, COMMAND_PROTOCOL, DigitalPy.service_id) + + def get_all_service_health(self): + """This method is used to get the health of all services""" + service_health = {} + for service_id, service_desc in self._service_index.items(): + if service_desc.status == ServiceStatus.RUNNING: + service_health[service_id] = self.get_service_health( + service_id) + + return service_health + + def get_service_health(self, service_id: str): + """This method is used to get the health of a service""" + req: Request = ObjectFactory.get_new_instance("Request") + req.set_action(COMMAND_ACTION) + req.set_context(service_id) + req.set_value("command", str(ServiceOperations.GET_HEALTH.value)) + req.set_value("target_service_id", service_id) + req.set_format("pickled") + self.subject_send_request(req, COMMAND_PROTOCOL, service_id) + resp = self.broker_receive_response(request_id=req.get_id(), timeout=5) + + if resp is None: + service_health: ServiceHealth = ServiceHealth() + service_health.status = ServiceHealthCategory.UNRESPONSIVE + service_health.service_id = service_id + service_health.timestamp = datetime.now() + + else: + service_health = resp.get_value("message") + + return service_health + + def listen_for_commands(self) -> List[Response]: + """this method is responsible for waiting for the response, enabling + the response to be sent by the main process responsible for sending + CoT's. This handler simply returns an empty list in the case that there is no + data to receive however if data is available from the /routing/response/ + topic it will be received parsed and returned so that it might be sent to + all clients by the main loop + """ + responses = self.broker_receive() + return responses + + def initialize_service_description(self, service_configuration: dict, + service_id: str) -> ServiceDescription: + """This method is used to initialize a service description""" + # construct the service description + new_service_instance = ServiceDescription() + + # set the id of the service + new_service_instance.id = service_id + + # add the values from the configuration to the service description + new_service_instance.name = str(service_configuration.get("name")) + new_service_instance.protocol = str( + service_configuration.get("protocol")) + new_service_instance.network_interface = str( + service_configuration.get("network")) + new_service_instance.status = ServiceStatus.STOPPED + new_service_instance.description = str( + service_configuration.get("description")) + new_service_instance.port = int( + service_configuration.get("port")) # type: ignore + new_service_instance.host = str(service_configuration.get("host")) + + # add the service description to the service index + self._service_index[new_service_instance.id] = new_service_instance + + return new_service_instance + + def start_service(self, service_id: str): + """This method is used to initialize the service process and start the service""" + + # check if the service is already running + if self.is_service_running(service_id): + return + + # parse the service id into the component name and the service name + component_name, service_name = service_id.split(".") + + # get the configuration for the service + service_component_configuration = self.component_index[component_name] + + # get the service configuration from the manifest of the component + service_configuration = service_component_configuration.get_section( + service_name) + + # initialize the service description + service_description = self.initialize_service_description( + service_configuration, service_id) + + # initialize the service class + service_class = self.initialize_service_class( + service_configuration, service_description) + + # start the service process + self.process_controller.start_process( + service_description, service_class) + + # update the status of the service + service_description.status = ServiceStatus.RUNNING + + def is_service_running(self, service_id: str) -> bool: + """This method is used to check if a service is running""" + + # if the service is not in the service index then it is not running + if service_id not in self._service_index: + return False + # if the service is in the service index and the status is running + # then the service is running, otherwise it is not running. + return self._service_index[service_id].status == ServiceStatus.RUNNING + + def initialize_service_class(self, service_configuration: dict, + service_description: ServiceDescription) -> DigitalPyService: + """This method is used to initialize a service class + + Args: + service_configuration (dict): + The configuration settings for the service. This is a dictionary that contains + key-value pairs of configuration settings. + service_description (ServiceDescription): + An object that describes the service. This includes properties like the name of the service, + its version, and other metadata. + + Returns: + DigitalPyService: an initialized digitalpy service object + """ + base_config: Configuration = ObjectFactory.get_instance( + "configuration") + + service_configuration.update(base_config.get_section("Service")) + + # initialize the service class + service_class: DigitalPyService = ObjectFactory.get_instance( + service_description.name, service_configuration) + + return service_class + + def stop_service(self, service_id: str): + """ + This method is used to stop a service. + + Args: + service_id (str): The ID of the service to stop. + + Notes: + - If the specified service ID matches the ID of the current service, the `stop()` + method will be called to handle a system shutdown. + - The method will send a service stop request using the `_send_service_stop_request()` + method. + - The `stop_process()` method of the process controller will be called to stop the + service process. + - The status of the service will be updated to `ServiceStatus.STOPPED`. + """ + if service_id.lower() == self.service_id.lower(): + self.stop() + + service_description = self._service_index[service_id] + + self._send_service_stop_request(service_id) + + self.process_controller.stop_process(service_description) + + service_description.status = ServiceStatus.STOPPED + + def _send_service_stop_request(self, service_id: str): + """ + This method is used to send a service stop request to the service. this method is used + internally as a helper method for the `stop_service()` method. + + Args: + service_id (str): The ID of the service to stop. + """ + req: Request = ObjectFactory.get_new_instance("Request") + req.set_action(COMMAND_ACTION) + req.set_context(service_id) + req.set_value("command", str(ServiceOperations.STOP.value)) + req.set_value("target_service_id", service_id) + req.set_format("pickled") + self.subject_send_request(req, COMMAND_PROTOCOL, service_id) diff --git a/digitalpy/core/service_management/controllers/service_management_process_controller.py b/digitalpy/core/service_management/controllers/service_management_process_controller.py new file mode 100644 index 00000000..2b9b83d9 --- /dev/null +++ b/digitalpy/core/service_management/controllers/service_management_process_controller.py @@ -0,0 +1,102 @@ +""" +Contains the ServiceManagementProcessController +class, which is responsible for managing the lifecycle of services in the application. +This includes starting and stopping service processes. + +The class inherits from the Controller class and overrides its methods to provide specific +functionality for service management. The start_process method is used to start a new service +process, while the stop_process method is used to stop a running service process. + +The ServiceManagementProcessController class uses the multiprocessing module to manage service +processes, and handles any exceptions that may occur during the start or stop operations. +It also uses the ObjectFactory class to get instances of required objects, and the +ServiceDescription class to describe the services to be managed. + +The file also imports several other modules and classes that are used in the +ServiceManagementProcessController class, including Request, Response, ActionMapper, Configuration, +DigitalPyService, and SERVICE_WAIT_TIME. +""" +import multiprocessing + +from digitalpy.core.digipy_configuration.configuration import Configuration +from digitalpy.core.main.controller import Controller +from digitalpy.core.main.object_factory import ObjectFactory +from digitalpy.core.service_management.digitalpy_service import DigitalPyService +from digitalpy.core.service_management.domain.service_description import ServiceDescription +from digitalpy.core.zmanager.action_mapper import ActionMapper +from digitalpy.core.zmanager.request import Request +from digitalpy.core.zmanager.response import Response + +from ..configuration.service_management_constants import SERVICE_WAIT_TIME + + +class ServiceManagementProcessController(Controller): + """ Service Management Process Controller class. Responsible for handling all operations on + a service's underlying process. + """ + + def __init__(self, request: Request, response: Response, sync_action_mapper: ActionMapper, + configuration: Configuration): + super().__init__(request, response, sync_action_mapper, configuration) + + def start_process(self, service: ServiceDescription, process_class: DigitalPyService): + """ + Starts a new process for the given service using the provided process class. + + Args: + service (ServiceDescription): The service description object. + process_class (DigitalPyService): The process class to be used for starting the service. + + Raises: + ChildProcessError: If the service fails to start or if post-processing fails. + + Returns: + None + """ + try: + process = multiprocessing.Process( + target=process_class.start, + args=( + ObjectFactory.get_instance("factory"), + ObjectFactory.get_instance("tracingprovider"), + service.host, + service.port + ) + ) + process.start() + except Exception as e: + raise ChildProcessError("Service failed to start") from e + try: + service.pid = process.pid + if process.is_alive(): + service.process = process + else: + raise ChildProcessError("Service failed to start") + except Exception as e: + process.terminate() + raise ChildProcessError( + "Service post-processing failed " + str(e)) from e + + def stop_process(self, service: ServiceDescription): + """ + Stops the specified service process. + + Args: + service (ServiceDescription): The service to stop. + + Raises: + ChildProcessException: If the service fails to stop. + + """ + try: + service.process.join(SERVICE_WAIT_TIME) + + # if the process is still alive, terminate it + if service.process.is_alive(): + service.process.terminate() + service.process.join() + + else: + return + except Exception as e: + raise ChildProcessError("Service failed to stop") from e diff --git a/digitalpy/core/service_management/controllers/service_management_sender_controller.py b/digitalpy/core/service_management/controllers/service_management_sender_controller.py index 1d5f66da..20eb335e 100644 --- a/digitalpy/core/service_management/controllers/service_management_sender_controller.py +++ b/digitalpy/core/service_management/controllers/service_management_sender_controller.py @@ -1,30 +1,33 @@ ####################################################### -# +# # core_name_controller.py # Python implementation of the Class service_management # Generated by Enterprise Architect # Created on: 16-Dec-2022 10:56:02 AM # Original author: Giu Platania -# +# ####################################################### -from typing import List, Union +from typing import List, Union, TYPE_CHECKING from digitalpy.core.main.object_factory import ObjectFactory from digitalpy.core.zmanager.request import Request from digitalpy.core.zmanager.response import Response -from digitalpy.core.domain.node import Node from digitalpy.core.main.controller import Controller -from digitalpy.core.IAM.IAM_facade import IAM + +if TYPE_CHECKING: + from digitalpy.core.IAM.IAM_facade import IAM + USER_DELIMITER = ";" + class ServiceManagementSenderController(Controller): """contains the business logic related to sending messages to services """ def __init__(self, request, response, service_management_action_mapper, sync_action_mapper, configuration): super().__init__(request, response, sync_action_mapper, configuration) - self.iam = IAM(sync_action_mapper, request, response, configuration) + self.iam: IAM = ObjectFactory.get_instance("IAM") def initialize(self, request: Request, response: Response): self.request = request @@ -47,40 +50,82 @@ def publish(self, recipients: Union[List[str], str], **kwargs) -> List[str]: List[str]: a list of topics to which the message should be published """ - main_topics = {} + main_topics: dict = {} return_topics: List[str] = [] + # case in which a specific set of users are meant to receive a given message if isinstance(recipients, list): - self.iam.get_connections_by_id(recipients) - for recipient_object in self.response.get_value("connections"): - recipient_main_topic = f"/{recipient_object.service_id}/{recipient_object.protocol}/{self.response.get_sender()}/{self.response.get_context()}/{self.response.get_action()}/{self.response.get_id()}/" - if recipient_main_topic in main_topics: - main_topics[recipient_main_topic]+= str(recipient_object.get_oid())+USER_DELIMITER - else: - main_topics[recipient_main_topic] = str(recipient_object.get_oid())+USER_DELIMITER + main_topics.update(self.create_user_specific_topic(recipients)) + # case in which the message should be sent to all recipients elif isinstance(recipients, str) and recipients == "*": - self.iam.get_all_connections() - for recipient_object in self.response.get_value("connections"): - recipient_main_topic = f"/{recipient_object.service_id}/{recipient_object.protocol}/{self.response.get_sender()}/{self.response.get_context()}/{self.response.get_action()}/{self.response.get_id()}/" - if recipient_main_topic not in main_topics: - main_topics[recipient_main_topic] = "" - + self.create_all_user_topic(main_topics) + + else: + raise ValueError( + "The recipients must be a list of recipients id's or a * representing that the message should be sent to all connected clients") + # iterate the main topics dictionary, # concatenate the ids and finally add them - # all into one list + # all into one list # TODO add memoization to prevent duplicate serialization of the same protocol for main_topic, ids in main_topics.items(): self.request.set_value("protocol", main_topic.split("/")[2]) sub_response = self.execute_sub_action("serialize") formatter = ObjectFactory.get_instance("formatter") formatter.serialize(sub_response) - return_topics.append(main_topic.encode()+ids.encode()+b","+sub_response.get_values()) + return_topics.append(main_topic.encode() + + ids.encode()+b","+sub_response.get_values()) if self.response.get_value("topics") is not None: self.response.get_value("topics").extend(return_topics) else: self.response.set_value("topics", return_topics) - def execute(self, method = None): + def create_all_user_topic(self, main_topics): + self.iam.get_all_connections() + + # filter the recipients based on the request + self.iam.filter_recipients( + self.request, self.response.get_value("connections")) + + for recipient_object in self.response.get_value("connections"): + recipient_main_topic = f"/{recipient_object.service_id}/{recipient_object.protocol}/{self.response.get_sender()}/{self.response.get_context()}/{self.response.get_action()}/{self.response.get_id()}/" + if recipient_main_topic not in main_topics: + main_topics[recipient_main_topic] = "" + + def create_user_specific_topic(self, recipients: list[str]) -> dict[str, str]: + """this method is used to create the topic to publish a message + + Args: + recipients (list[str]): a list of recipients id's + + Returns: + dict[str, str]: a dictionary containing the main topic as key and the concatenated ids as value + """ + topics = {} + + # get the connection objects from the IAM + self.iam.get_connections_by_id(recipients) + + # filter the recipients based on the request + self.iam.filter_recipients( + recipients=self.response.get_value("connections")) + + # iterate the recipients and create the main topic + for recipient_object in self.response.get_value("connections"): + + recipient_main_topic = f"/{recipient_object.service_id}/{recipient_object.protocol}/{self.response.get_sender()}/{self.response.get_context()}/{self.response.get_action()}/{self.response.get_id()}/" + # if the main topic is already present in the dictionary, concatenate the connection id + if recipient_main_topic in topics: + topics[recipient_main_topic] += str( + recipient_object.get_oid())+USER_DELIMITER + # if the main topic is not present in the dictionary, add it with the connection id + else: + topics[recipient_main_topic] = str( + recipient_object.get_oid())+USER_DELIMITER + + return topics + + def execute(self, method=None): pass diff --git a/digitalpy/core/service_management/digitalpy_service.py b/digitalpy/core/service_management/digitalpy_service.py index 7f641778..fc311c38 100644 --- a/digitalpy/core/service_management/digitalpy_service.py +++ b/digitalpy/core/service_management/digitalpy_service.py @@ -1,22 +1,96 @@ +""" +This file defines a class `DigitalPyService` that inherits from `Service`, `ZmqSubscriber`, +and `ZMQPusher`. This class is used to create a service that can subscribe to messages, push messages, +and perform service-related operations. + +The class constructor takes several parameters including service id, addresses, ports, protocols, +a formatter, and a network interface. It initializes the parent classes and sets up various +properties. + +The class has several properties with their respective getters and setters, such as +`protocol`, `status`, and `tracer`. + +The class also defines several methods, some of which are abstract and must be implemented +by any class that inherits from `DigitalPyService`. These include `event_loop`, +`handle_command`, and `handle_exception`. + +The `start` method is used to start the service. It configures the object factory, +initializes the tracer, initializes the controllers, and starts the event loop. +If a network is provided, it also initializes the network. + +The `__getstate__` and `__setstate__` methods are used for pickling and unpickling the object, +respectively. +""" ####################################################### -# +# # DigitalPyService.py # Python implementation of the Class DigitalPyService # Generated by Enterprise Architect # Created on: 02-Dec-2022 5:39:44 PM # Original author: Giu Platania -# +# ####################################################### +from datetime import datetime +import traceback +from typing import List + +from digitalpy.core.domain.domain.service_health import ServiceHealth +from digitalpy.core.main.impl.default_factory import DefaultFactory +from digitalpy.core.main.object_factory import ObjectFactory +from digitalpy.core.service_management.domain.service_operations import ServiceOperations +from digitalpy.core.service_management.domain.service_status import ServiceStatus +from digitalpy.core.telemetry.tracing_provider import TracingProvider +from digitalpy.core.telemetry.tracer import Tracer from digitalpy.core.zmanager.service import Service from digitalpy.core.zmanager.impl.zmq_subscriber import ZmqSubscriber from digitalpy.core.zmanager.impl.zmq_pusher import ZMQPusher from digitalpy.core.parsing.formatter import Formatter +from digitalpy.core.network.network_interface import NetworkInterface +from digitalpy.core.zmanager.response import Response +from digitalpy.core.IAM.IAM_facade import IAM +from digitalpy.core.domain.domain.network_client import NetworkClient +from digitalpy.core.zmanager.request import Request +from digitalpy.core.health.domain.service_health_category import ServiceHealthCategory +from digitalpy.core.digipy_configuration.configuration import Configuration + +COMMAND_PROTOCOL = "command" +COMMAND_ACTION = "ServiceCommand" + class DigitalPyService(Service, ZmqSubscriber, ZMQPusher): - # on the reception of messages from the subscriber interface or the socket - #TODO: what is the service manager supposed to do? is this going to be a new service - - def __init__(self, service_id: str, subject_address: str, subject_port: int, subject_protocol: str, integration_manager_address: str, integration_manager_port: int, integration_manager_protocol: str, formatter: Formatter): + """ + Represents a DigitalPy service. + + This class is responsible for managing the lifecycle and behavior of a DigitalPy service. + It inherits from the Service, ZmqSubscriber, and ZMQPusher classes. + + Args: + service_id (str): The unique ID of the service inheriting from DigitalPyService. + subject_address (str): The address of the zmanager "subject". + subject_port (int): The port of the zmanager "subject". + subject_protocol (str): The protocol of the zmanager "subject". + integration_manager_address (str): The address of the zmanager "integration_manager". + integration_manager_port (int): The port of the zmanager "integration_manager". + integration_manager_protocol (str): The protocol of the zmanager "integration_manager". + formatter (Formatter): The formatter used by the service to serialize the request + values to and from messages. + network (NetworkInterface): The network interface used by the service to send and + receive messages. + protocol (str): The protocol of the service. + error_threshold (float, optional): The error threshold for the service. Defaults to 0.1. + """ + + def __init__(self, service_id: str, + subject_address: str, + subject_port: int, + subject_protocol: str, + integration_manager_address: str, + integration_manager_port: int, + integration_manager_protocol: str, + formatter: Formatter, + network: NetworkInterface, + protocol: str, + error_threshold: float = 0.1): """the constructor for the digitalpy service class Args: @@ -28,6 +102,8 @@ def __init__(self, service_id: str, subject_address: str, subject_port: int, sub integration_manager_port (int): the port of the zmanager "integration_manager" integration_manager_protocol (str): the protocol of the zmanager "integration_manager" formatter (Formatter): the formatter used by the service to serialize the request values to and from messages, (should be injected by object factory) + network (NetworkInterface): the network interface used by the service to send and receive messages, (should be injected by object factory through the services' constructor) + protocol (str): the protocol of the service """ Service.__init__(self) ZmqSubscriber.__init__(self, formatter) @@ -39,6 +115,109 @@ def __init__(self, service_id: str, subject_address: str, subject_port: int, sub self.integration_manager_port = integration_manager_port self.integration_manager_protocol = integration_manager_protocol self.service_id = service_id + self._tracer = None + self.status = ServiceStatus.UNKNOWN + self.network = network + self.protocol = protocol + self.response_queue: List[Response] = [] + self.iam_facade: IAM = ObjectFactory.get_instance("IAM") + + self.total_requests = 0 + self.total_errors = 0 + self.total_request_processing_time = 0 + self.error_threshold = error_threshold + + def handle_connection(self, client: NetworkClient, req: Request): + """register a client with the server. This method should be called when a client connects to the server + so that it can be registered with the IAM component. + Args: + client (NetworkClient): the client to register + req (Request): the request from the client containing connection data + """ + + resp = ObjectFactory.get_new_instance("Response") + client.service_id = self.service_id + client.protocol = self.protocol + self.iam_facade.initialize(req, resp) + req.set_value("connection", client) + self.iam_facade.execute("connection") + return + + def handle_disconnection(self, client: NetworkClient, req: Request): + """unregister a client from the server. This method should be called when a client disconnects from the server + so that it can be unregistered from the IAM component. + Args: + client (NetworkClient): the client to unregister + req (Request): the request from the client containing disconnection data + """ + resp = ObjectFactory.get_new_instance("Response") + req.set_value("connection_id", str(client.get_oid())) + self.iam_facade.initialize(req, resp) + self.iam_facade.execute("disconnection") + return + + @property + def protocol(self) -> str: + """get the protocol of the service + + Returns: + str: the protocol of the service + """ + return self._protocol + + @protocol.setter + def protocol(self, protocol: str): + """set the protocol of the service + + Args: + protocol (str): the protocol of the service + """ + self._protocol = protocol + + @property + def status(self) -> ServiceStatus: + """get the status of the service + + Returns: + ServiceStatus: the status of the service + """ + return self._status + + @status.setter + def status(self, status: ServiceStatus): + self._status = status + + @property + def tracer(self) -> Tracer: + """get the tracer of the service + + Raises: + ValueError: if the tracer has not been initialized it will raise a value error + + Returns: + Tracer: the tracer of the service + """ + if self._tracer is None: + raise ValueError("Tracer has not been initialized") + else: + return self._tracer + + @tracer.setter + def tracer(self, value: Tracer): + """set the tracer of the service + + Args: + value (Tracer): the tracer of the service + + Raises: + ValueError: if the tracer has already been initialized it will raise a value error + ValueError: if the value is not an instance of Tracer it will raise a value error + """ + if self._tracer is not None: + raise ValueError("Tracer has already been initialized") + elif not isinstance(value, Tracer): + raise ValueError("Tracer must be an instance of TracingProvider") + self._tracer = value def discovery(self): """used to inform the discoverer of the specifics of this service""" @@ -51,20 +230,186 @@ def send_heart_beat(self): # TODO: once the service manager has been well defined then we will need # to define the format for this service. + def initialize_controllers(self): + """used to initialize the controllers once the service is started. Should be overriden + by inheriting classes + """ + def initialize_connections(self, application_protocol: str): - """initialize connections to the subject and the integration manager + """initialize connections to the subject and the integration manager within the + zmanager architecture. The topic subscribed to by the subject is as follows: + /messages/ + /commands/ Args: application_protocol (str): the application protocol of the service """ - ZMQPusher.initiate_connections(self, self.subject_port, self.subject_address, self.service_id) - self.broker_connect(self.integration_manager_address, self.integration_manager_port, self.integration_manager_protocol, self.service_id, application_protocol) + ZMQPusher.initiate_connections( + self, self.subject_port, self.subject_address, self.service_id) + self.broker_connect(self.integration_manager_address, self.integration_manager_port, + self.integration_manager_protocol, self.service_id, + application_protocol) + + def response_handler(self, responses: List[Response]): + """used to handle a response. Should be overriden by inheriting classes""" + for response in responses: + if response.get_action() == COMMAND_ACTION: + self.handle_command(response) + else: + self.handle_response(response) + + def handle_response(self, response: Response): + """used to handle a response. Should be overriden by inheriting classes""" + if self.network: + if response.get_value("recipients") == "*": + self.network.send_message_to_all_clients(response) + else: + self.network.send_message_to_clients( + response, response.get_value("recipients")) + + def event_loop(self): + """used to run the service. Should be overriden by inheriting classes""" + if self.network: + self.handle_network() + + responses = self.broker_receive() + self.response_handler(responses) + + def handle_network(self): + """used to handle the network.""" + requests = self.network.service_connections() + for request in requests: + self.handle_inbound_message(request) + + def stop(self): + """ + Stops the service by performing necessary cleanup operations. + + This method sets the service status to STOPPING, tears down the network if it exists, + disconnects the broker, tears down connections, sets the service status to STOPPED, + and raises SystemExit to exit the program. + """ + self.status = ServiceStatus.STOPPING + + if self.network: + self.network.teardown_network() + + self.broker_disconnect() + self.teardown_connections() + + self.status = ServiceStatus.STOPPED + + raise SystemExit + + def handle_inbound_message(self, message: Request) -> bool: + """This function is used to handle inbound messages from other services. + It is intiated by the event loop. + + Args: + message (Request): the message to handle + + Returns: + bool: True if the message was handled successfully, False otherwise + """ + + # TODO: discuss this with giu and see if we should move the to the action mapping system? + if message.get_value("action") == "connection": + self.handle_connection(message.get_value("client"), message) + return True + + elif message.get_value("action") == "disconnection": + self.handle_disconnection(message.get_value("client"), message) + return True + + return False + + def handle_command(self, command: Response): + """used to handle a command. Should be overriden by inheriting classes""" + if command.get_value("command") == ServiceOperations.STOP.value: + self.stop() + elif command.get_value("command") == ServiceOperations.GET_HEALTH.value: + service_health = self.get_health() + conf: Configuration = ObjectFactory.get_instance("Configuration") + resp = ObjectFactory.get_new_instance("Response") + resp.set_value("message", service_health) + resp.set_action("publish") + resp.set_format("pickled") + resp.set_id(command.get_id()) + self.subject_send_request(resp, COMMAND_PROTOCOL, conf.get_value( + "service_id", "ServiceManager")) + + def get_health(self): + """used to get the health of the service.""" + service_health: ServiceHealth = ServiceHealth() + if self.total_requests == 0: + service_health.error_percentage = 0 + service_health.average_request_time = 0 + else: + service_health.error_percentage = self.total_errors/self.total_requests + service_health.average_request_time = self.total_request_processing_time/self.total_requests + service_health.service_id = self.service_id + service_health.status = self.calculate_health(service_health) + service_health.timestamp = datetime.now() + return service_health + + def calculate_health(self, service_health: ServiceHealth): + """used to calculate the health of the service.""" + if service_health.error_percentage > self.error_threshold: + return ServiceHealthCategory.DEGRADED + else: + return ServiceHealthCategory.OPERATIONAL + + def handle_exception(self, exception: Exception): + """This function is used to handle exceptions that occur in the service. + It is intiated by the event loop. + """ + if isinstance(exception, SystemExit): + self.status = ServiceStatus.STOPPED + raise SystemExit + else: + traceback.print_exc() + print("An exception occurred: " + str(exception)) + self.total_errors += 1 + + def start(self, object_factory: DefaultFactory, tracing_provider: TracingProvider, host: str = "", port: int = 0): + """used to start the service and initialize the network if provided + + Args: + object_factory (DefaultFactory): the object factory used to create instances of objects + tracing_provider (TracingProvider): the tracing provider used to create a tracer + host (str, optional): the host of the network. Defaults to "". + port (int, optional): the port of the network. Defaults to 0. + """ + + ObjectFactory.configure(object_factory) + self.tracer = tracing_provider.create_tracer(self.service_id) + self.initialize_controllers() + self.initialize_connections(self.protocol) + + if self.network and host and port: + self.network.intialize_network(host, port) + + elif self.network: + raise ValueError( + "network has been injected but host and port have not been provided") + self.status = ServiceStatus.RUNNING + self.execute_main_loop() + + def execute_main_loop(self): + """used to execute the main loop of the service""" + while self.status == ServiceStatus.RUNNING: + try: + self.event_loop() + except Exception as ex: + self.handle_exception(ex) + if self.status == ServiceStatus.STOPPED: + exit(0) def __getstate__(self): ZmqSubscriber.__getstate__(self) ZMQPusher.__getstate__(self) return self.__dict__ - + def __setstate__(self, state): ZmqSubscriber.__setstate__(self, state) - ZMQPusher.__setstate__(self, self.__dict__) \ No newline at end of file + ZMQPusher.__setstate__(self, self.__dict__) diff --git a/digitalpy/core/service_management/domain/__init__.py b/digitalpy/core/service_management/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/digitalpy/core/service_management/domain/service_description.py b/digitalpy/core/service_management/domain/service_description.py new file mode 100644 index 00000000..bbdf1b27 --- /dev/null +++ b/digitalpy/core/service_management/domain/service_description.py @@ -0,0 +1,291 @@ +from datetime import datetime as dt +from multiprocessing import Process +from digitalpy.core.domain.node import Node +from digitalpy.core.service_management.domain.service_status import ServiceStatus + +class ServiceDescription(Node): + def __init__(self, node_type = "service_description", oid=None) -> None: + super().__init__(node_type, oid=oid) + # the pid of the service, changing depending on the status of the service + self._pid: int = None # type: ignore + # the id of the service (must be unique), consistent throughout the lifetime of the service, this must be in the form, . + self._id: str = None # type: ignore + # the protocol of the service + self._protocol: str = None # type: ignore + # the status of the service in it's lifecycle + self._status: ServiceStatus = None # type: ignore + # the description of the service + self._description: str = None # type: ignore + # the human readable name of the service, should be unique but not required. + self._name: str = None # type: ignore + # when the last message from the service was received + self._last_message_time: dt = None # type: ignore + # when the service was started + self._start_time: dt = None # type: ignore + # the process of the service (if it is running) + self._process: Process = None # type: ignore + # host address if the process is attached to a network + self._host: str = None # type: ignore + # port if the process is attached to a network + self._port: int = None # type: ignore + # network interface if the process is attached to a network + self._network_interface: str = None # type: ignore + + @property + def network_interface(self) -> str: + """get the network interface of the service + + Returns: + str: the network interface of the service + """ + return self._network_interface + + @network_interface.setter + def network_interface(self, network_interface: str): + """set the network interface of the service + + Args: + network_interface (str): the network interface of the service + + Raises: + TypeError: if network_interface is not a string + """ + if not isinstance(network_interface, str): + raise TypeError("'network_interface' must be a string") + + self._network_interface = network_interface + + @property + def host(self) -> str: + """get the host of the service + + Returns: + str: the host of the service + """ + return self._host + + @host.setter + def host(self, host: str): + """set the host of the service + + Args: + host (str): the host of the service + + Raises: + TypeError: if host is not a string + """ + if not isinstance(host, str): + raise TypeError("'host' must be a string") + self._host = host + + @property + def port(self) -> int: + """get the port of the service + + Returns: + int: the port of the service + """ + return self._port + + @port.setter + def port(self, port: int): + """set the port of the service + + Args: + port (int): the port of the service + + Raises: + TypeError: if port is not an integer + """ + if not isinstance(port, int): + raise TypeError("'port' must be an integer") + self._port = port + + @property + def process(self) -> Process: + """get the process of the service + + Returns: + Process: the process of the service + """ + return self._process + + @process.setter + def process(self, process: Process): + """set the process of the service + + Args: + process (Process): the process of the service + """ + if not isinstance(process, Process): + raise TypeError("'process' must be an instance of Process") + if not process.is_alive(): + raise ValueError("'process' must be alive") + if self._process != None: + raise ValueError("'process' must be None to set to a new value") + self._process = process + + @property + def start_time(self) -> dt: + """get the start time of the service + + Returns: + dt: the start time of the service + """ + return self._start_time + + @start_time.setter + def start_time(self, start_time: dt): + """set the start time of the service + + Args: + start_time (dt): the start time of the service + """ + if not isinstance(start_time, dt): + raise TypeError("'start_time' must be an instance of datetime") + self._start_time = start_time + + @property + def last_message_time(self) -> dt: + """get the last message time of the service + + Returns: + dt: the last message time of the service + """ + return self._last_message_time + + @last_message_time.setter + def last_message_time(self, last_message_time: dt): + """set the last message time of the service + + Args: + last_message_time (dt): the last message time of the service + """ + if not isinstance(last_message_time, dt): + raise TypeError("'last_message_time' must be an instance of datetime") + self._last_message_time = last_message_time + + @property + def pid(self) -> int: + """get the pid of the service + + Returns: + int: the pid of the service + """ + return self._pid + + @pid.setter + def pid(self, pid: int): + """set the pid of the service, this should only be done by the service manager + and only the in the event the service is restarted. + + Args: + pid (int): the pid of the service + """ + if not isinstance(pid, int): + raise TypeError("'pid' must be an integer") + self._pid = pid + + @property + def id(self) -> str: + """get the id of the service + + Returns: + str: the id of the service + """ + return self._id + + @id.setter + def id(self, id: str): + """set the id of the service. + + Args: + id (str): the id of the service + """ + if not isinstance(id, str): + raise TypeError("'id' must be a string") + self._id = id + + @property + def protocol(self) -> str: + """get the protocol of the service + + Returns: + str: the protocol of the service + """ + return self._protocol + + @protocol.setter + def protocol(self, protocol: str): + """set the protocol of the service + + Args: + protocol (str): the protocol of the service + """ + if not isinstance(protocol, str): + raise TypeError("'protocol' must be a string") + + self._protocol = protocol + + @property + def status(self) -> ServiceStatus: + """get the status of the service + + Returns: + ServiceStatus: the status of the service + """ + return self._status + + @status.setter + def status(self, status: ServiceStatus): + """set the status of the service + + Args: + status (ServiceStatus): the status of the service + """ + if not isinstance(status, ServiceStatus): + raise TypeError("'status' must be an instance of ServiceStatus") + + self._status = status + + @property + def description(self) -> str: + """get the description of the service + + Returns: + str: the description of the service + """ + return self._description + + @description.setter + def description(self, description: str): + """set the description of the service + + Args: + description (str): the description of the service + """ + if not isinstance(description, str): + raise TypeError("'description' must be a string") + + self._description = description + + @property + def name(self) -> str: + """get the name of the service + + Returns: + str: the name of the service + """ + return self._name + + @name.setter + def name(self, name: str): + """set the name of the service + + Args: + name (str): the name of the service + """ + if not isinstance(name, str): + raise TypeError("'name' must be a string") + + self._name = name \ No newline at end of file diff --git a/digitalpy/core/service_management/domain/service_manager_operations.py b/digitalpy/core/service_management/domain/service_manager_operations.py new file mode 100644 index 00000000..dd635a24 --- /dev/null +++ b/digitalpy/core/service_management/domain/service_manager_operations.py @@ -0,0 +1,37 @@ +""" +This module defines the ServiceManagerOperations Enum class which represents the different +operations that can be performed by a service manager. + +Classes: + ServiceManagerOperations: An enumeration of the different operations that can be + performed by a service manager. +""" + +from enum import Enum + + +class ServiceManagerOperations(Enum): + """ + A class used to represent the different operations that can be performed by a service manager. + + ... + + Attributes + ---------- + START_SERVICE : str + Represents the operation to start a service. + STOP_SERVICE : str + Represents the operation to stop a service. + RESTART_SERVICE : str + Represents the operation to restart a service. + GET_ALL_SERVICE_HEALTH : str + Represents the operation to get the health status of all services. + + Methods + ------- + None + """ + START_SERVICE = "start_service" + STOP_SERVICE = "stop_service" + RESTART_SERVICE = "restart_service" + GET_ALL_SERVICE_HEALTH = "get_all_service_health" diff --git a/digitalpy/core/service_management/domain/service_operations.py b/digitalpy/core/service_management/domain/service_operations.py new file mode 100644 index 00000000..00a22273 --- /dev/null +++ b/digitalpy/core/service_management/domain/service_operations.py @@ -0,0 +1,30 @@ +""" +This module defines the ServiceOperations Enum class which represents the different operations +that can be performed on a service. + +Classes: + ServiceOperations: An enumeration of the different operations + that can be performed on a service. +""" + +from enum import Enum + +class ServiceOperations(Enum): + """ + A class used to represent the different operations that can be performed on a service. + + ... + + Attributes + ---------- + STOP : str + Represents the operation to stop a service. + STATUS : str + Represents the operation to check the health status of a service. + + Methods + ------- + None + """ + GET_HEALTH = "get_health" + STOP = "stop_service" \ No newline at end of file diff --git a/digitalpy/core/service_management/domain/service_status.py b/digitalpy/core/service_management/domain/service_status.py new file mode 100644 index 00000000..44ffd4ca --- /dev/null +++ b/digitalpy/core/service_management/domain/service_status.py @@ -0,0 +1,13 @@ + +from enum import Enum + +class ServiceStatus(Enum): + RUNNING = "Running" + # a state where the service is completely stopped + STOPPED = "Stopped" + # an intermediate state between running and stopped to allow for a clean shutdown + STOPPING = "Stopping" + PAUSED = "Paused" + UNKNOWN = "Unknown" + ERROR = "Error" + DEAD = "Dead" diff --git a/digitalpy/core/service_management/service_management_facade.py b/digitalpy/core/service_management/service_management_facade.py index ecbd376f..0be547e1 100644 --- a/digitalpy/core/service_management/service_management_facade.py +++ b/digitalpy/core/service_management/service_management_facade.py @@ -64,7 +64,7 @@ def __init__( # instantiating the sync_action_mapper for use by the service management sender controller to call external sync_action_mapper = ObjectFactory.get_instance("syncactionmapper") self.sender_controller = ServiceManagementSenderController(request, response, service_management_action_mapper, sync_action_mapper, configuration) - + def initialize(self, request, response): self.request = request self.response = response @@ -85,3 +85,8 @@ def execute(self, method): def publish(self, *args, **kwargs): self.sender_controller.publish(*args, **kwargs) + + def start_service(self, *args, **kwargs): + """This method is used to initialize the service process and start the service + """ + self.service_m \ No newline at end of file diff --git a/digitalpy/core/telemetry/tracer.py b/digitalpy/core/telemetry/tracer.py index 83426bfb..14d32b38 100644 --- a/digitalpy/core/telemetry/tracer.py +++ b/digitalpy/core/telemetry/tracer.py @@ -1,9 +1,9 @@ from abc import ABC, abstractmethod - +from typing import Any class Tracer(ABC): @abstractmethod - def start_as_current_span(self, span_name: str): + def start_as_current_span(self, span_name: str) -> Any: """this really isnt an abstractmethod however the telemetry system needs to move forward so at least this way refrences can be identified and replaced when/if the time comes that opentelemetry is no longer the target telemetry diff --git a/digitalpy/core/telemetry/tracing_provider.py b/digitalpy/core/telemetry/tracing_provider.py index d8f222f5..64fea7ef 100644 --- a/digitalpy/core/telemetry/tracing_provider.py +++ b/digitalpy/core/telemetry/tracing_provider.py @@ -3,6 +3,11 @@ class TracingProvider(ABC): + @abstractmethod + def initialize_tracing(self): + """initialize the tracing provider + """ + @abstractmethod def create_tracer(self, tracer_name: str) -> Tracer: """create a new tracer instance from the current tracer provider diff --git a/digitalpy/core/testing/digitalpy_test_runner.py b/digitalpy/core/testing/digitalpy_test_runner.py new file mode 100644 index 00000000..960b9be8 --- /dev/null +++ b/digitalpy/core/testing/digitalpy_test_runner.py @@ -0,0 +1,15 @@ +from multiprocessing import Process +from digitalpy.core.main.DigitalPy import DigitalPy + + +def start_digitalpy_application(app: type[DigitalPy]): + """Starts the DigitalPy application.""" + app().start(True) + + +def stop_digitalpy_application(app: Process): + """Stops the DigitalPy application.""" + app.join(3) + if app.is_alive(): + app.terminate() + app.join() diff --git a/digitalpy/core/testing/network_tcp_test_client.py b/digitalpy/core/testing/network_tcp_test_client.py new file mode 100644 index 00000000..8fd2ddf6 --- /dev/null +++ b/digitalpy/core/testing/network_tcp_test_client.py @@ -0,0 +1,27 @@ +"""A test client for the network_tcp module.""" + +import socket + + +class NetworkTCPTestClient: + """A test client for the network_tcp module.""" + + def __init__(self, host: str, port: int): + self.host: str = host + self.port: int = port + self.conn: socket.socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM) + self.conn.connect((host, port)) + + def send(self, data: bytes) -> None: + """Send data to the server.""" + self.conn.send(data) + + def recv(self, size: int, timeout: int = 5) -> bytes: + """Receive data from the server.""" + self.conn.settimeout(timeout) + return self.conn.recv(size) + + def close(self) -> None: + """Close the connection.""" + self.conn.close() diff --git a/digitalpy/core/zmanager/controller_message.py b/digitalpy/core/zmanager/controller_message.py index a7c80daf..7d95f674 100644 --- a/digitalpy/core/zmanager/controller_message.py +++ b/digitalpy/core/zmanager/controller_message.py @@ -1,14 +1,17 @@ from abc import ABC, abstractmethod +from typing import Any + + class ControllerMessage(ABC): - + @abstractmethod def get_id(self): return self.id - + @abstractmethod def set_id(self, id): self.id = id - + @abstractmethod def set_sender(self, sender): self.sender = sender @@ -16,11 +19,11 @@ def set_sender(self, sender): @abstractmethod def get_sender(self): return self.sender - + @abstractmethod def set_context(self, context): self.context = context - + @abstractmethod def get_context(self): return self.context @@ -32,26 +35,26 @@ def get_action(self): @abstractmethod def set_action(self, action): self.action = action - + @abstractmethod def set_value(self, name, value): """Set a value @param name The name of the variable @param value The value of the variable """ - + @abstractmethod def set_values(self, values: dict): """set all key value pairs at once""" - + @abstractmethod - def get_values(self): + def get_values(self) -> dict[Any, Any]: """get all key value pairs at once""" - + @abstractmethod - def get_value(self, name, default=None): + def get_value(self, name, default=None) -> Any: """Get a value""" - + @abstractmethod def get_boolean_value(self, name, default=False): """Get a value as boolean""" @@ -59,19 +62,19 @@ def get_boolean_value(self, name, default=False): @abstractmethod def clear_value(self, name): """Remove a value""" - + @abstractmethod def clear_values(self): """Remove all values""" - + @abstractmethod def has_value(self, name): """Check for existence of a value""" - + @abstractmethod def set_property(self, name, value): """Set a property""" - + @abstractmethod def get_property(self, name): - """Get a property""" \ No newline at end of file + """Get a property""" diff --git a/digitalpy/core/zmanager/impl/default_request.py b/digitalpy/core/zmanager/impl/default_request.py index 72b56052..608f24db 100644 --- a/digitalpy/core/zmanager/impl/default_request.py +++ b/digitalpy/core/zmanager/impl/default_request.py @@ -49,4 +49,5 @@ def get_method(self) -> str: """ return self.method - \ No newline at end of file + def set_format(self, format_: str): + self.format = format_ \ No newline at end of file diff --git a/digitalpy/core/zmanager/impl/default_response.py b/digitalpy/core/zmanager/impl/default_response.py index d7fa6e79..043d5085 100644 --- a/digitalpy/core/zmanager/impl/default_response.py +++ b/digitalpy/core/zmanager/impl/default_response.py @@ -36,4 +36,4 @@ def get_status(self): """Gets the status of this response object. Returns: int: the status code of this response object""" - return self.status \ No newline at end of file + return self.status diff --git a/digitalpy/core/zmanager/impl/default_routing_worker.py b/digitalpy/core/zmanager/impl/default_routing_worker.py index 27123038..76d26648 100644 --- a/digitalpy/core/zmanager/impl/default_routing_worker.py +++ b/digitalpy/core/zmanager/impl/default_routing_worker.py @@ -16,7 +16,7 @@ from digitalpy.core.parsing.formatter import Formatter from digitalpy.core.main.factory import Factory from digitalpy.core.telemetry.impl.opentel_metrics_provider import OpenTelMetricsProvider - +from digitalpy.core.service_management.digitalpy_service import COMMAND_PROTOCOL class DefaultRoutingWorker: def __init__( @@ -124,8 +124,14 @@ def start(self): try: protocol, request = self.receive_request() service_id = request.get_value("service_id") - response = self.process_request(protocol, request) - self.send_response(response, protocol=protocol, service_id=service_id) + # if the protocol is COMMAND_PROTOCOL, then the request is a command to a service and should + # be sent directly to integration_manager so that it can be processed by the service + if protocol == COMMAND_PROTOCOL: + self.send_response(request, protocol, request.get_value("service_id")) + + else: + response = self.process_request(protocol, request) + self.send_response(response, protocol=protocol, service_id=service_id) except Exception as ex: try: self.send_error(ex) diff --git a/digitalpy/core/zmanager/impl/zmq_pusher.py b/digitalpy/core/zmanager/impl/zmq_pusher.py index e9a42482..54b3d4eb 100644 --- a/digitalpy/core/zmanager/impl/zmq_pusher.py +++ b/digitalpy/core/zmanager/impl/zmq_pusher.py @@ -10,9 +10,9 @@ def __init__(self, formatter: Formatter): # list of connection to which the socket should reconnect after # being unpickled self.__pusher_socket_connections = [] - self.pusher_context = None - self.pusher_socket = None - self.pusher_formatter = formatter + self.pusher_context: zmq.Context = None # type: ignore + self.pusher_socket: zmq.Socket = None # type: ignore + self.pusher_formatter: Formatter = formatter self.logger = logging.getLogger(self.__class__.__name__) def initiate_connections(self, port: int, address: str, service_id: str): @@ -33,6 +33,17 @@ def initiate_connections(self, port: int, address: str, service_id: str): # add the connection to the connections list so it can be reconnected upon serialization self.__pusher_socket_connections.append(f"tcp://{address}:{port}") + def teardown_connections(self): + """teardown subject connection + """ + for connection in self.__pusher_socket_connections: + self.pusher_socket.disconnect(connection) + self.pusher_socket.close() + self.pusher_context.term() + self.pusher_context.destroy() + self.pusher_socket = None # type: ignore + self.pusher_context = None # type: ignore + def subject_bind(self, address: int, port: str): """create the ZMQ zocket @@ -42,14 +53,13 @@ def subject_bind(self, address: int, port: str): """ self.pusher_socket.connect(f"tcp://{address}:{port}") - def subject_send_request(self, request: Request, protocol: str, service_id: str = None): + def subject_send_request(self, request: Request, protocol: str, service_id: str = None): # type: ignore """send the message to a Puller Args: request (Request): the request to be sent to the subject protocol (str): the protocol of the request to be sent - service_id (str): the id of the service to which the response - of the message is expected to be sent + service_id (str, optional): the service_id of the request to be sent. Defaults to the id of the current service. """ if service_id is None: service_id = self.service_id diff --git a/digitalpy/core/zmanager/impl/zmq_subscriber.py b/digitalpy/core/zmanager/impl/zmq_subscriber.py index ce1696ef..85c990d7 100644 --- a/digitalpy/core/zmanager/impl/zmq_subscriber.py +++ b/digitalpy/core/zmanager/impl/zmq_subscriber.py @@ -1,5 +1,6 @@ +import time import zmq -from typing import List +from typing import List, Union import logging from digitalpy.core.zmanager.subscriber import Subscriber @@ -8,6 +9,8 @@ from digitalpy.core.parsing.formatter import Formatter RECIPIENT_DELIMITER = ";" + + class ZmqSubscriber(Subscriber): # 1. Create a context # 2. Create a socket @@ -21,42 +24,100 @@ def __init__(self, formatter: Formatter): # being unpickled self.__subscriber_socket_connections = [] - self.subscriber_context = None - self.subscriber_socket = None + self.subscriber_context: zmq.Context = None # type: ignore + self.subscriber_socket: zmq.Socket = None # type: ignore self.subscriber_formatter = formatter self.logger = logging.getLogger(self.__class__.__name__) + self.responses: list[str] = [] def broker_connect(self, integration_manager_address: str, integration_manager_port: int, integration_manager_protocol: str, service_identity: str, application_protocol: str): """Connect or reconnect to broker """ - + if not isinstance(integration_manager_address, str): raise TypeError("'integration_manager_address' must be a string") if not isinstance(integration_manager_port, int): raise TypeError("'integration_manager_port' must be an integer") if not isinstance(integration_manager_protocol, str): raise TypeError("'integration_manager_protocol' must be a string") - + self.service_identity = service_identity - # added to fix hanging connect issue as per + # added to fix hanging connect issue as per # https://stackoverflow.com/questions/44257579/zeromq-hangs-in-a-python-multiprocessing-class-object-solution if self.subscriber_context == None: self.subscriber_context = zmq.Context() if self.subscriber_socket == None: self.subscriber_socket = self.subscriber_context.socket(zmq.SUB) - self.__subscriber_socket_connections.append(f"{integration_manager_protocol}://{integration_manager_address}:{integration_manager_port}") + self.__subscriber_socket_connections.append( + f"{integration_manager_protocol}://{integration_manager_address}:{integration_manager_port}") # add the connection to the connections list so it can be reconnected upon serialization - self.subscriber_socket.connect(f"{integration_manager_protocol}://{integration_manager_address}:{integration_manager_port}") - self.subscriber_socket.setsockopt_string(zmq.SUBSCRIBE, f"/messages/{service_identity}/") + self.subscriber_socket.connect( + f"{integration_manager_protocol}://{integration_manager_address}:{integration_manager_port}") + self.subscriber_socket.setsockopt_string( + zmq.SUBSCRIBE, f"/messages/{service_identity}/") # currently nothing is done with the commands endpoint but it will be used in future - self.subscriber_socket.setsockopt_string(zmq.SUBSCRIBE, f"/commands/{service_identity}/") + self.subscriber_socket.setsockopt_string( + zmq.SUBSCRIBE, f"/commands/{service_identity}/") # unlimited as trunkating can result in unsent data and broken messages # TODO: determine a sane default self.subscriber_socket.setsockopt(zmq.RCVHWM, 0) self.subscriber_socket.setsockopt(zmq.SNDHWM, 0) - - def broker_receive(self, blocking: bool=False, max_messages: int = 100) -> List[Response]: + + def broker_disconnect(self): + """Disconnect from broker + """ + for connection in self.__subscriber_socket_connections: + self.subscriber_socket.disconnect(connection) + self.subscriber_socket.close() + self.subscriber_context.term() + self.subscriber_context.destroy() + + def broker_receive_response(self, request_id: str, blocking: bool = True, timeout=1) -> Union[Response, None]: + """Returns the reply message or None if there was no reply""" + try: + s_time = time.time() + while s_time + timeout > time.time(): + if not blocking: + message = self.subscriber_socket.recv_multipart(flags=zmq.NOBLOCK)[ + 0].split(b" ", 1) + else: + self.subscriber_socket.setsockopt( + zmq.RCVTIMEO, timeout*1000) + message = self.subscriber_socket.recv_multipart()[ + 0].split(b" ", 1) + # instantiate the response object + response: Response = ObjectFactory.get_new_instance("response") + + # TODO: this is assuming that the message from the integration manager is pickled + response.set_format("pickled") + + # get the values returned from the routing proxy and serialize them to + values = message[1].strip(b",") + response.set_values(values) + self.subscriber_formatter.deserialize(response) + + topic = message[0] + decoded_topic = topic.decode("utf-8") + topic_sections = decoded_topic.split("/") + _, _, service_id, protocol, sender, context, action, id, recipients = topic_sections + response.set_sender(sender) + response.set_context(context) + response.set_action(action) + response.set_id(id) + + if len(recipients.split(RECIPIENT_DELIMITER)) > 1: + response.set_value("recipients", recipients.split( + RECIPIENT_DELIMITER)[:-1]) + if response.get_id() == request_id: + return response + else: + self.responses.append(response) + return None + except zmq.ZMQError as ex: + return None + + def broker_receive(self, blocking: bool = False, max_messages: int = 100) -> List[Response]: """Returns the reply message or None if there was no reply Args: blocking (False): whether or not the operation is blocking. Option defaults to False. @@ -64,16 +125,20 @@ def broker_receive(self, blocking: bool=False, max_messages: int = 100) -> List[ """ responses = [] try: + for _ in self.responses: + responses.append(self.responses.pop()) # TODO: move the range to a configuration file # this protects against the case where messages are being sent faster than they can be received for _ in range(max_messages): if not blocking: - message = self.subscriber_socket.recv_multipart(flags=zmq.NOBLOCK)[0].split(b" ", 1) - else: - message = self.subscriber_socket.recv_multipart()[0].split(b" ", 1) + message = self.subscriber_socket.recv_multipart(flags=zmq.NOBLOCK)[ + 0].split(b" ", 1) + else: + message = self.subscriber_socket.recv_multipart()[ + 0].split(b" ", 1) # instantiate the response object - response = ObjectFactory.get_new_instance("response") - + response: Response = ObjectFactory.get_new_instance("response") + # TODO: this is assuming that the message from the integration manager is pickled response.set_format("pickled") @@ -91,10 +156,12 @@ def broker_receive(self, blocking: bool=False, max_messages: int = 100) -> List[ response.set_action(action) response.set_id(id) - if len(recipients.split(RECIPIENT_DELIMITER))>1: - response.set_value("recipients", recipients.split(RECIPIENT_DELIMITER)[:-1]) + if len(recipients.split(RECIPIENT_DELIMITER)) > 1: + response.set_value("recipients", recipients.split( + RECIPIENT_DELIMITER)[:-1]) responses.append(response) + return responses except zmq.ZMQError as ex: return responses @@ -103,7 +170,7 @@ def broker_send(self, message): """Send request to broker """ self.subscriber_socket.send(message) - + def __getstate__(self): state = self.__dict__ if "subscriber_socket" in state: @@ -111,12 +178,14 @@ def __getstate__(self): if "subscriber_context" in state: del state["subscriber_context"] return state - + def __setstate__(self, state): self.__dict__ = state self.subscriber_context = zmq.Context() self.subscriber_socket = self.subscriber_context.socket(zmq.SUB) for connection in self.__subscriber_socket_connections: self.subscriber_socket.connect(connection) - self.subscriber_socket.setsockopt_string(zmq.SUBSCRIBE, f"/messages/{self.service_identity}/") - self.subscriber_socket.setsockopt_string(zmq.SUBSCRIBE, f"/commands/{self.service_identity}/") \ No newline at end of file + self.subscriber_socket.setsockopt_string( + zmq.SUBSCRIBE, f"/messages/{self.service_identity}/") + self.subscriber_socket.setsockopt_string( + zmq.SUBSCRIBE, f"/commands/{self.service_identity}/") diff --git a/digitalpy/core/zmanager/integration_manager.py b/digitalpy/core/zmanager/integration_manager.py index 4dbdc5cf..fc292562 100644 --- a/digitalpy/core/zmanager/integration_manager.py +++ b/digitalpy/core/zmanager/integration_manager.py @@ -1,6 +1,6 @@ ################ # Author: FreeTAKTeam -# The Integration manager receives all answers from all workers, prints them, and sends a message +# The Integration manager receives all answers from all workers, prints them, and sends a message # to the workers to shut down when all tasks are complete. # Uses a ZMQ_PULL socket to receive answers from the workers. # Uses a ZMQ_PUB socket to send the FINISH message to the workers. @@ -8,8 +8,10 @@ ################ +from typing import List import zmq + class IntegrationManager: def __init__(self, integration_manager_puller_protocol: str, integration_manager_puller_address: str, integration_manager_puller_port: int, integration_manager_publisher_protocol: str, integration_manager_publisher_address: str, integration_manager_publisher_port: int) -> None: @@ -20,7 +22,6 @@ def __init__(self, integration_manager_puller_protocol: str, integration_manager self.integration_manager_publisher_address = integration_manager_publisher_address self.integration_manager_publisher_port = integration_manager_publisher_port - def initialize_connections(self): context = zmq.Context() @@ -29,7 +30,8 @@ def initialize_connections(self): # unlimited as trunkating can result in unsent data and broken messages # TODO: determine a sane default pull_socket.setsockopt(zmq.RCVHWM, 0) - pull_socket.bind(f"{self.integration_manager_puller_protocol}://{self.integration_manager_puller_address}:{self.integration_manager_puller_port}") + pull_socket.bind( + f"{self.integration_manager_puller_protocol}://{self.integration_manager_puller_address}:{self.integration_manager_puller_port}") self.pull_socket = pull_socket # create a pub socket @@ -37,7 +39,8 @@ def initialize_connections(self): # unlimited as trunkating can result in unsent data and broken messages # TODO: determine a sane default pub_socket.setsockopt(zmq.SNDHWM, 0) - pub_socket.bind(f"{self.integration_manager_publisher_protocol}://{self.integration_manager_publisher_address}:{self.integration_manager_publisher_port}") + pub_socket.bind( + f"{self.integration_manager_publisher_protocol}://{self.integration_manager_publisher_address}:{self.integration_manager_publisher_port}") self.pub_socket = pub_socket def start(self): @@ -47,14 +50,19 @@ def start(self): while True: try: # receive a message from a client - request = self.pull_socket.recv_multipart()[0] - response_protocol, response_object_unserialized = request.split(b',', 1) + request_bytes: list[bytes] = self.pull_socket.recv_multipart() + + request = request_bytes[0] + + response_protocol, response_object_unserialized = request.split( + b',', 1) subject = b"/messages" + response_protocol try: # send the response back to the client - self.pub_socket.send(subject + b" " + response_object_unserialized) + self.pub_socket.send( + subject + b" " + response_object_unserialized) except Exception as ex: print("Error sending response to client: {}".format(ex)) except Exception as ex: - print("Error "+str(ex)) \ No newline at end of file + print("Error "+str(ex)) diff --git a/digitalpy/core/zmanager/request.py b/digitalpy/core/zmanager/request.py index ed4f112c..14aab5d1 100644 --- a/digitalpy/core/zmanager/request.py +++ b/digitalpy/core/zmanager/request.py @@ -1,9 +1,10 @@ -from abc import ABC, abstractmethod +from abc import abstractmethod from digitalpy.core.zmanager.response import Response from digitalpy.core.zmanager.controller_message import ControllerMessage + class Request(ControllerMessage): @abstractmethod @@ -17,3 +18,7 @@ def get_response(self): @abstractmethod def get_method(self): raise NotImplementedError + + @abstractmethod + def set_format(self, format_: str): + raise NotImplementedError diff --git a/digitalpy/core/zmanager/routing_proxy.py b/digitalpy/core/zmanager/routing_proxy.py index a7d7af07..8d124685 100644 --- a/digitalpy/core/zmanager/routing_proxy.py +++ b/digitalpy/core/zmanager/routing_proxy.py @@ -31,7 +31,6 @@ def start_workers(self): self.workers.append(worker_process) def initiate_sockets(self): - print("initiate_sockets") self.context = zmq.Context() self.backend_dealer = self.context.socket(zmq.DEALER) self.backend_dealer.bind(self.backend_address) diff --git a/digitalpy/core/zmanager/service.py b/digitalpy/core/zmanager/service.py index 952cbc40..ca4f771d 100644 --- a/digitalpy/core/zmanager/service.py +++ b/digitalpy/core/zmanager/service.py @@ -1,5 +1,8 @@ from abc import ABC, abstractmethod +from digitalpy.core.main.impl.default_factory import DefaultFactory +from digitalpy.core.telemetry.tracing_provider import TracingProvider + class Service(ABC): def __init__(self): self.running = True @@ -12,9 +15,10 @@ def discovery(self): @abstractmethod def send_heart_beat(self): """send a heartbeat to inform the the service manager that it's alive""" - + # TODO: this is seemingly replaced by the implementation of the health check + @abstractmethod - def start(self): + def start(self, object_factory: DefaultFactory, tracing_provider: TracingProvider): """this method should be used to start the service as a process""" @abstractmethod diff --git a/digitalpy/core/zmanager/subject.py b/digitalpy/core/zmanager/subject.py index c7d524b3..85166aab 100644 --- a/digitalpy/core/zmanager/subject.py +++ b/digitalpy/core/zmanager/subject.py @@ -34,7 +34,6 @@ def start_workers(self): self.workers.append(worker_process) def initiate_sockets(self): - print("initiate_sockets") self.context = zmq.Context() self.backend_pusher = self.context.socket(zmq.PUSH) self.backend_pusher.bind(self.backend_address) @@ -54,7 +53,7 @@ def begin_routing(self): self.logger.debug("receieved %s",str(message)) self.backend_pusher.send_multipart(message) except Exception as ex: - self.logger.fatal("exception thrown in subject %s", ex, exc_info=1) + self.logger.fatal("exception thrown in subject %s", ex, exc_info=True) def __getstate__(self): """delete objects that cannot be pickled or generally serialized""" diff --git a/pyproject.toml b/pyproject.toml index 9c8616eb..eb4e6aca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "DigitalPy" -version = "0.3.13.7" +version = "0.3.14" description="A python implementation of the aphrodite's specification, heavily based on WCMF" authors = ["FreeTAKTeam "] packages = [ @@ -11,6 +11,9 @@ include = ["digitalpy/**/*.json", "digitalpy/**/*.py", "digitalpy/**/*.ini", "di [tool.poetry.dependencies] rule-engine="*" pyzmq="*" +opentelemetry-sdk="*" +lxml="*" +xmltodict="*" [tool.poetry.extras] dev = ["pytest"]