Merge pull request #247 from Matthijsy/hotfix/15m-data

Parse data based on 15M data, and allow for configuring averaging to 60M

Author: JaccoR

custom_components/entsoe/__init__.py

@@ -20,6 +20,7 @@
     DEFAULT_MODIFYER,
     DEFAULT_ENERGY_SCALE,
     DOMAIN,
+    CONF_PERIOD,
 )
 from .coordinator import EntsoeCoordinator
 from .services import async_setup_services
@@ -42,6 +43,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
     # Initialise the coordinator and save it as domain-data
     api_key = entry.options[CONF_API_KEY]
     area = entry.options[CONF_AREA]
+    period = entry.options.get(CONF_PERIOD, "PT60M")
     energy_scale = entry.options.get(CONF_ENERGY_SCALE, DEFAULT_ENERGY_SCALE)
     modifyer = entry.options.get(CONF_MODIFYER, DEFAULT_MODIFYER)
     vat = entry.options.get(CONF_VAT_VALUE, 0)
@@ -52,6 +54,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
         hass,
         api_key=api_key,
         area=area,
+        period=period,
         energy_scale=energy_scale,
         modifyer=modifyer,
         calculation_mode=calculation_mode,

custom_components/entsoe/api_client.py

@@ -3,23 +3,29 @@
 import enum
 import logging
 import xml.etree.ElementTree as ET
+from collections import defaultdict
 from datetime import datetime, timedelta
 from typing import Dict, Union
 
 import pytz
 import requests
 
+from custom_components.entsoe.const import DEFAULT_PERIOD
+from custom_components.entsoe.utils import get_interval_minutes
+from .utils import bucket_time
+
 _LOGGER = logging.getLogger(__name__)
 URL = "https://web-api.tp.entsoe.eu/api"
 DATETIMEFORMAT = "%Y%m%d%H00"
 
 
 class EntsoeClient:
 
-    def __init__(self, api_key: str):
+    def __init__(self, api_key: str, period: str = DEFAULT_PERIOD) -> None:
         if api_key == "":
             raise TypeError("API key cannot be empty")
         self.api_key = api_key
+        self.configuration_period = period
 
     def _base_request(
         self, params: Dict, start: datetime, end: datetime
@@ -48,7 +54,7 @@ def _remove_namespace(self, tree):
 
     def query_day_ahead_prices(
         self, country_code: Union[Area, str], start: datetime, end: datetime
-    ) -> str:
+    ) -> dict:
         """
         Parameters
         ----------
@@ -74,14 +80,16 @@ def query_day_ahead_prices(
                 return dict(sorted(series.items()))
 
             except Exception as exc:
-                _LOGGER.debug(f"Failed to parse response content:{response.content}")
+                _LOGGER.debug(
+                    f"Failed to parse response content error: {exc} content:{response.content}"
+                )
                 raise exc
         else:
-            print(f"Failed to retrieve data: {response.status_code}")
+            _LOGGER.error(f"Failed to retrieve data: {response.status_code}")
             return None
 
     # lets process the received document
-    def parse_price_document(self, document: str) -> str:
+    def parse_price_document(self, document: str) -> dict:
 
         root = self._remove_namespace(ET.fromstring(document))
         _LOGGER.debug(f"content: {root}")
@@ -125,65 +133,64 @@ def parse_price_document(self, document: str) -> str:
                     )
                     continue
 
-                if resolution == "PT60M":
-                    series.update(self.process_PT60M_points(period, start_time))
-                else:
-                    series.update(self.process_PT15M_points(period, start_time))
-
-                # Now fill in any missing hours
-                current_time = start_time
-                last_price = series[current_time]
-
-                while current_time < end_time:  # upto excluding! the endtime
-                    if current_time in series:
-                        last_price = series[current_time]  # Update to the current price
-                    else:
-                        _LOGGER.debug(
-                            f"Extending the price {last_price} of the previous hour to {current_time}"
-                        )
-                        series[current_time] = (
-                            last_price  # Fill with the last known price
-                        )
-                    current_time += timedelta(hours=1)
-
+                # Parse the resolution, we only support the 'PTxM' format
+                interval = get_interval_minutes(resolution)
+                data = self.process_points(period, start_time, interval)
+                if resolution != self.configuration_period:
+                    _LOGGER.debug(
+                        f"Got {interval} minutes interval prices, but period is configured on {self.configuration_period} minutes. Averaging data into intervals of {self.configuration_period} minutes."
+                    )
+                    data = self.average_to_interval(
+                        data,
+                        expected_interval=get_interval_minutes(
+                            self.configuration_period
+                        ),
+                    )
+                series.update(data)
         return series
 
     # processing hourly prices info -> thats easy
-    def process_PT60M_points(self, period: Element, start_time: datetime):
-        data = {}
-        for point in period.findall(".//Point"):
-            position = point.find(".//position").text
-            price = point.find(".//price.amount").text
-            hour = int(position) - 1
-            time = start_time + timedelta(hours=hour)
-            data[time] = float(price)
-        return data
+    def process_points(
+        self, period: Element, start_time: datetime, interval: int
+    ) -> dict:
+        _LOGGER.debug(f"Processing prices based on interval {interval} minutes")
+        # Extract (position, price) pairs
+        points = sorted(
+            (int(p.findtext(".//position")), float(p.findtext(".//price.amount")))
+            for p in period.findall(".//Point")
+        )
+        if not points:
+            return {}
 
-    # processing quarterly prices -> this is more complex
-    def process_PT15M_points(self, period: Element, start_time: datetime):
-        positions = {}
+        data = {}
+        last_price = None
+        for pos in range(points[0][0], points[-1][0] + 1):
+            if points and pos == points[0][0]:
+                last_price = points.pop(0)[1]
+            data[start_time + timedelta(minutes=(pos - 1) * interval)] = last_price
 
-        # first store all positions
-        for point in period.findall(".//Point"):
-            position = point.find(".//position").text
-            price = point.find(".//price.amount").text
-            positions[int(position)] = float(price)
+        return data
 
-        # now calculate hourly averages based on available points
-        data = {}
-        last_hour = (max(positions.keys()) + 3) // 4
-        last_price = 0
+    def average_to_interval(self, data: dict, expected_interval: int) -> dict:
+        """
+        Average prices into the expected interval buckets
 
-        for hour in range(last_hour):
-            sum_prices = 0
-            for idx in range(hour * 4 + 1, hour * 4 + 5):
-                last_price = positions.get(idx, last_price)
-                sum_prices += last_price
+        args:
+            data: The data to average
+            expected_interval: The interval in minutes after transformation (e.g. 30, 60)
+        """
 
-            time = start_time + timedelta(hours=hour)
-            data[time] = round(sum_prices / 4, 2)
+        # Create buckets of expected_interval
+        by_hour = defaultdict(list)
+        for timestamp, price in data.items():
+            bucket = bucket_time(timestamp, expected_interval)
+            by_hour[bucket].append(price)
 
-        return data
+        # Calculate the average for each bucket
+        return {
+            hour: round(sum(prices) / len(prices), 2)
+            for hour, prices in by_hour.items()
+        }
 
 
 class Area(enum.Enum):

custom_components/entsoe/config_flow.py

@@ -38,6 +38,9 @@
     DOMAIN,
     ENERGY_SCALES,
     UNIQUE_ID,
+    CONF_PERIOD,
+    PERIOD_OPTIONS,
+    DEFAULT_PERIOD,
 )
 
 
@@ -102,6 +105,7 @@ async def async_step_user(
                     options={
                         CONF_API_KEY: user_input[CONF_API_KEY],
                         CONF_AREA: user_input[CONF_AREA],
+                        CONF_PERIOD: user_input[CONF_PERIOD],
                         CONF_MODIFYER: user_input[CONF_MODIFYER],
                         CONF_CURRENCY: user_input[CONF_CURRENCY],
                         CONF_ENERGY_SCALE: user_input[CONF_ENERGY_SCALE],
@@ -129,6 +133,9 @@ async def async_step_user(
                             ]
                         ),
                     ),
+                    vol.Required(CONF_PERIOD): SelectSelector(
+                        SelectSelectorConfig(options=PERIOD_OPTIONS),
+                    ),
                     vol.Optional(CONF_ADVANCED_OPTIONS, default=False): bool,
                 },
             ),
@@ -316,7 +323,9 @@ async def async_step_init(
                     ): vol.All(vol.Coerce(float, "must be a number")),
                     vol.Optional(
                         CONF_MODIFYER,
-                        description={"suggested_value": self.config_entry.options[CONF_MODIFYER]},
+                        description={
+                            "suggested_value": self.config_entry.options[CONF_MODIFYER]
+                        },
                         default=DEFAULT_MODIFYER,
                     ): TemplateSelector(TemplateSelectorConfig()),
                     vol.Optional(
@@ -331,6 +340,12 @@ async def async_step_init(
                             CONF_ENERGY_SCALE, DEFAULT_ENERGY_SCALE
                         ),
                     ): vol.In(list(ENERGY_SCALES.keys())),
+                    vol.Optional(
+                        CONF_PERIOD,
+                        default=self.config_entry.options.get(
+                            CONF_PERIOD, DEFAULT_PERIOD
+                        ),
+                    ): vol.In(PERIOD_OPTIONS),
                     vol.Optional(
                         CONF_CALCULATION_MODE,
                         default=calculation_mode_default,

custom_components/entsoe/const.py

@@ -8,6 +8,7 @@
 CONF_API_KEY = "api_key"
 CONF_ENTITY_NAME = "name"
 CONF_AREA = "area"
+CONF_PERIOD = "period"
 CONF_MODIFYER = "modifyer"
 CONF_CURRENCY = "currency"
 CONF_ENERGY_SCALE = "energy_scale"
@@ -18,6 +19,7 @@
 DEFAULT_MODIFYER = "{{current_price}}"
 DEFAULT_CURRENCY = CURRENCY_EURO
 DEFAULT_ENERGY_SCALE = "kWh"
+DEFAULT_PERIOD = "PT60M"
 
 # default is only for internal use / backwards compatibility
 CALCULATION_MODE = {
@@ -27,7 +29,8 @@
     "publish": "publish",
 }
 
-ENERGY_SCALES = { "kWh": 1000, "MWh": 1 }
+ENERGY_SCALES = {"kWh": 1000, "MWh": 1}
+PERIOD_OPTIONS = ["PT60M", "PT15M"]
 
 # Commented ones are not working at entsoe
 AREA_INFO = {