Merge pull request #16 from codegreen-framework/main

0.0.3 version

Author: shubhvjain

.gitignore

@@ -177,3 +177,4 @@ poetry.lock
 
 codegreen_core/tools/test.py
 codegreen_core/data/test.py
+tests/test_notebook.ipynb

README.md

@@ -2,6 +2,16 @@
 
 This repository contains the main functionality of the codegreen project. The complete documentation including installation and usage are available on the [documentation website](https://codegreen-framework.github.io/codegreen-core/). 
 
+# Development 
+
+## Installation
+- `git clone`
+- install poetry
+- install in editable mode : `poetry install`
+
+## Github workflows
+Changes in the repo also triggers github actions 
+
 ## Development workflow 
 - the `release` branch contains the latest stable version of the released python package 
 - the `main` branch contains stable, tested code ready to be released. 

codegreen_core/__init__.py

@@ -1,3 +1,5 @@
-from .utilities.config import Config
-
-Config.load_config()
+from . import utilities
+from . import data
+from . import tools
+from . import models
+utilities.config.Config.load_config()

codegreen_core/data/entsoe.py

@@ -117,17 +117,23 @@ def _refine_data(options, data1):
     data1.sort_index(inplace=True)
     return {"data": data1, "refine_logs": refine_logs}
 
+def _convert_local_to_utc(dte):
+    # datetime obj is converted from local time zone to utc 
+    local_timezone = datetime.now().astimezone().tzinfo
+    return pd.Timestamp(dte,tz=local_timezone).tz_convert('UTC')
 
 def _entsoe_get_actual_generation(options={"country": "", "start": "", "end": ""}):
     """Fetches the aggregated actual generation per production type data (16.1.B&C) for the given country within the given start and end date
     params: options = {country (2 letter country code),start,end} . Both the dates are in the YYYYMMDDhhmm format and the local time zone
     returns : {"data":pd.DataFrame, "duration":duration (in min) of the time series data, "refine_logs":"notes on refinements made" }
     """
+    utc_start = _convert_local_to_utc(options["start"]) 
+    utc_end =   _convert_local_to_utc(options["end"]) 
     client1 = entsoePandas(api_key=_get_API_token())
     data1 = client1.query_generation(
         options["country"],
-        start=pd.Timestamp(options["start"], tz="UTC"),
-        end=pd.Timestamp(options["end"], tz="UTC"),
+        start = utc_start ,
+        end = utc_end ,
         psr_type=None,
     )
     # drop columns with actual consumption values (we want actual aggregated generation values)
@@ -159,8 +165,8 @@ def _entsoe_get_total_forecast(options={"country": "", "start": "", "end": ""}):
     client = entsoePandas(api_key=_get_API_token())
     data = client.query_generation_forecast(
         options["country"],
-        start=pd.Timestamp(options["start"], tz="UTC"),
-        end=pd.Timestamp(options["end"], tz="UTC"),
+        start=_convert_local_to_utc(options["start"]) ,
+        end=_convert_local_to_utc(options["end"]) 
     )
     # if the data is a series instead of a dataframe, it will be converted to a dataframe
     if isinstance(data, pd.Series):
@@ -188,8 +194,8 @@ def _entsoe_get_wind_solar_forecast(options={"country": "", "start": "", "end":
     client = entsoePandas(api_key=_get_API_token())
     data = client.query_wind_and_solar_forecast(
         options["country"],
-        start=pd.Timestamp(options["start"], tz="UTC"),
-        end=pd.Timestamp(options["end"], tz="UTC"),
+        start=_convert_local_to_utc(options["start"]) ,
+        end=_convert_local_to_utc(options["end"]) 
     )
     durationMin = (data.index[1] - data.index[0]).total_seconds() / 60
     # refining the data
@@ -225,6 +231,10 @@ def _convert_to_60min_interval(rawData):
         # determining how many rows need to be combined to get data in 60 min format.
         groupingFactor = int(60 / duration)
         oldData = rawData["data"]
+        # check if there is enough data to convert to 60 min
+        if (len(oldData) < groupingFactor):
+            raise ValueError("Data cannot be converted into 60 min interval since there is inadequate number of rows in the data")
+        
         oldData["startTimeUTC"] = pd.to_datetime(oldData["startTimeUTC"])
         start_time = oldData["startTimeUTC"].min()
         end_time = oldData["startTimeUTC"].max()
@@ -246,9 +256,19 @@ def _convert_to_60min_interval(rawData):
 
 
 def _convert_date_to_entsoe_format(dt: datetime):
+    """ rounds the date to nearest hour """
     return dt.replace(minute=0, second=0, microsecond=0).strftime("%Y%m%d%H%M")
 
 
+def _format_energy_data(df):
+    start_time_column = df.pop("startTimeUTC")
+    df.insert(0, "startTime", start_time_column)
+    local_timezone = datetime.now().astimezone().tzinfo
+    df["startTime"] = pd.to_datetime(df["startTime"], format="%Y%m%d%H%M").dt.tz_localize("UTC").dt.tz_convert(local_timezone)
+    df.insert(1, "startTimeUTC", start_time_column)
+    return df
+
+
 # the main methods
 
 
@@ -260,6 +280,7 @@ def get_actual_production_percentage(country, start, end, interval60=False) -> d
     :param str country: The 2 alphabet country code.
     :param datetime start: The start date for data retrieval. A Datetime object. Note that this date will be rounded to the nearest hour.
     :param datetime end: The end date for data retrieval. A datetime object. This date is also rounded to the nearest hour.
+    :param boolean interval60: To convert the data into 60 min time interval. False by default 
     :return: A DataFrame containing the hourly energy production mix and percentage of energy generated from renewable and non renewable sources.
     :return: A dictionary containing:
       - `error`: A string with an error message, empty if no errors.
@@ -269,12 +290,32 @@ def get_actual_production_percentage(country, start, end, interval60=False) -> d
     :rtype: dict
     """
     try:
+        if not isinstance(country, str):
+            raise ValueError("Invalid country")
+        if not isinstance(start, datetime):
+            raise ValueError("Invalid start date")
+        if not isinstance(end, datetime):
+            raise ValueError("Invalid end date")
+
+        if start > datetime.now():
+            raise ValueError("Invalid start date. Generation data is only available for the past and not the future. Use the forecast API instead")
+
+        if start > end :
+            raise ValueError("Invalid date range. End date must be greater than the start date")
+        
+        # if end date is in the future and the start date is in the past , only data till the available moment will be returned.
+        if end > datetime.now():
+            raise ValueError("Invalid end date. Generation data is only available for the past and not the future. Use the forecast API instead")
+            # this is not allowed because the entsoe-py returns error if it's greater than the present
+            #warnings.warn("End date is in the future. Will fetch data only till the present")
+
         options = {
             "country": country,
-            "start": start,
-            "end": end,
+            "start": start.replace(minute=0,second=0),
+            "end": end.replace(second=0,minute=0),
             "interval60": interval60,
         }
+        # print(options)
         # get actual generation data per production type and convert it into 60 min interval if required
         totalRaw = _entsoe_get_actual_generation(options)
         total = totalRaw["data"]
@@ -327,18 +368,18 @@ def get_actual_production_percentage(country, start, end, interval60=False) -> d
             table[fieldName] = table[fieldName].astype(int)
 
         return {
-            "data": table,
+            "data": _format_energy_data(table),
             "data_available": True,
-            "time_interval": totalRaw["duration"],
+            "time_interval": duration,
         }
     except Exception as e:
-        print(e)
+        # print(e)
         print(traceback.format_exc())
         return {
             "data": None,
             "data_available": False,
-            "error": Exception,
-            "time_interval": totalRaw["duration"],
+            "error": e,
+            "time_interval": 0,
         }
 
 
@@ -364,6 +405,13 @@ def get_forecast_percent_renewable(
     """
     try:
         # print(country,start,end)
+        if not isinstance(country, str):
+            raise ValueError("Invalid country")
+        if not isinstance(start, datetime):
+            raise ValueError("Invalid start date")
+        if not isinstance(end, datetime):
+            raise ValueError("Invalid end date")
+        
         start = _convert_date_to_entsoe_format(start)
         end = _convert_date_to_entsoe_format(end)
         options = {"country": country, "start": start, "end": end}
@@ -390,7 +438,7 @@ def get_forecast_percent_renewable(
             windsolar["startTimeUTC"], format="%Y%m%d%H%M"
         )
         windsolar["posix_timestamp"] = windsolar["startTimeUTC"].astype(int) // 10**9
-        return {"data": windsolar, "data_available": True, "time_interval": 60}
+        return {"data": _format_energy_data(windsolar), "data_available": True, "time_interval": 60}
     except Exception as e:
         print(e)
         print(traceback.format_exc())

codegreen_core/data/main.py

@@ -3,10 +3,12 @@
 
 from ..utilities.message import Message, CodegreenDataError
 from ..utilities import metadata as meta
-from . import entsoe as et
+from ..utilities.config import Config
 
+from . import entsoe as et
+from . import offline as off
 
-def energy(country, start_time, end_time, type="generation", interval60=True) -> dict:
+def energy(country, start_time, end_time, type="generation") -> dict:
     """
     Returns hourly time series of energy production mix for a specified country and time range.
 
@@ -19,8 +21,9 @@ def energy(country, start_time, end_time, type="generation", interval60=True) ->
      ========================== ========== ================================================================
       Column                     type       Description
      ========================== ========== ================================================================
-      startTimeUTC               datetime   Start date in UTC (60 min interval)
-      Biomass                    float64
+      startTimeUTC               object    Start date in UTC (format YYYYMMDDhhmm)
+      startTime                  datetime  Start time in local timezone
+      Biomass                    float64 
       Fossil Hard coal           float64
       Geothermal                 float64
       ....more energy sources    float64
@@ -47,11 +50,13 @@ def energy(country, start_time, end_time, type="generation", interval60=True) ->
     :param datetime start_time: The start date for data retrieval. A Datetime object. Note that this date will be rounded to the nearest hour.
     :param datetime end_time: The end date for data retrieval. A datetime object. This date is also rounded to the nearest hour.
     :param str type: The type of data to retrieve; either 'generation' or 'forecast'. Defaults to 'generation'.
+    :param boolean interval60: To fix the time interval of data to 60 minutes. True by default. Only applicable for generation data
+
     :return: A dictionary containing:
       - `error`: A string with an error message, empty if no errors.
       - `data_available`: A boolean indicating if data was successfully retrieved.
       - `data`: A pandas DataFrame containing the energy data if available, empty DataFrame if not.
-      - `time_interval` : the time interval of the DataFrame
+      - `time_interval` : the time interval of the DataFrame 
     :rtype: dict
     """
     if not isinstance(country, str):
@@ -70,11 +75,27 @@ def energy(country, start_time, end_time, type="generation", interval60=True) ->
     e_source = meta.get_country_energy_source(country)
     if e_source == "ENTSOE":
         if type == "generation":
-            return et.get_actual_production_percentage(
-                country, start_time, end_time, interval60
-            )
+            """
+            let local_found= false
+            see if caching is enabled, if yes, first check in the cache
+            if not, 
+                check if offline data is enabled
+                if yes, check is data is available locally 
+                if no, go online 
+            """
+            offline_data = off.get_offline_data(country,start_time,end_time)
+            if offline_data["available"] is True and offline_data["partial"] is False and offline_data["data"] is not None:
+                # todo fix this if partial get remaining data and merge instead of fetching the complete data
+                return {"data":offline_data["data"],"data_available":True,"error":"None","time_interval":60,"source":offline_data["source"]}
+            else:
+                energy_data = et.get_actual_production_percentage(country, start_time, end_time, interval60=True)
+                energy_data["data"] = energy_data["data"]
+                energy_data["source"] = "public_data"
+                return energy_data            
         elif type == "forecast":
-            return et.get_forecast_percent_renewable(country, start_time, end_time)
+            energy_data = et.get_forecast_percent_renewable(country, start_time, end_time)
+            energy_data["data"] = energy_data["data"]
+            return energy_data
     else:
         raise CodegreenDataError(Message.NO_ENERGY_SOURCE)
     return None