Merge pull request #13 from JasperE84/dev

v1.0.6 fixes faulty config parsing of cron values and adds more descriptive exception messages when kiosk api query fails

Author: JasperE84

README.md

@@ -122,6 +122,8 @@ Result:
 # Changelog
 | Version | Description |
 | --- | --- |
+| 1.0.5 | Fixed a bug  parsing the environment cron settings, which are in string format, but were interpreted as int, causing an exception |
+| 1.0.5 | FusionSolar API will now immediately be queried on startup if debug mode is enabled (so no waiting for cron to trigger is required for testing) |
 | 1.0.5 | Added InfluxDB support for an optional secondary grid telemetry EAN configuration (pvoutput output is only supported on the primary EAN) |
 | 1.0.5 | Bugfix for InfluxDB v1 implementation and removed auto-database creation for VictoriaMetrics compatibility |
 | 1.0.3 | Grid transformer usage measurement polling from Kenter's meetdata.nl API has been implemented |

gridrelay.py

@@ -24,19 +24,27 @@ def __init__(self, conf: PvConf, logger):
     def start(self):
         self.logger.debug("GridRelay waiting 5sec to initialize docker-compose containers")
         time.sleep(5)
+        
+        daystobackfill = self.conf.gridrelaydaystobackfill
 
         while 1:
             try:
-                grid_measurement_data = self.gridkenter.fetch_gridkenter_data(self.conf.gridrelaysysname, self.conf.gridrelaykenterean, self.conf.gridrelaykentermeterid, self.conf.gridrelaydaysback)
-                self.write_gridkenter_to_influxdb(grid_measurement_data)
-                self.write_gridkenter_to_pvoutput(grid_measurement_data)
-
-                if self.conf.gridrelaysys02enabled:
-                    grid_measurement_data = self.gridkenter.fetch_gridkenter_data(self.conf.gridrelaysysname02, self.conf.gridrelaykenterean02, self.conf.gridrelaykentermeterid02, self.conf.gridrelaydaysback)
+                for daysback in range(self.conf.gridrelaydaysback, self.conf.gridrelaydaysback + 1 + daystobackfill):
+                    grid_measurement_data = self.gridkenter.fetch_gridkenter_data(self.conf.gridrelaysysname, self.conf.gridrelaykenterean, self.conf.gridrelaykentermeterid, daysback)
                     self.write_gridkenter_to_influxdb(grid_measurement_data)
-                    #No support for pvoutput on 2 EAN codes yet (needs summing of kenter data or pvoutput support for 2 distinct systems)
-                    #self.write_gridkenter_to_pvoutput(grid_measurement_data)
+                    self.write_gridkenter_to_pvoutput(grid_measurement_data)
+
+                    if self.conf.gridrelaysys02enabled:
+                        grid_measurement_data = self.gridkenter.fetch_gridkenter_data(self.conf.gridrelaysysname02, self.conf.gridrelaykenterean02, self.conf.gridrelaykentermeterid02, daysback)
+                        self.write_gridkenter_to_influxdb(grid_measurement_data)
+                        #No support for pvoutput on 2 EAN codes yet (needs summing of kenter data or pvoutput support for 2 distinct systems)
+                        #self.write_gridkenter_to_pvoutput(grid_measurement_data)
+
+                    # Wait 5 secs for next backfill day
+                    if daystobackfill > 0: time.sleep(5);
 
+                # Don't backfill after initial backfill
+                daystobackfill = 0
             except:
                 self.logger.exception(
                     "Uncaught exception in GridRelay data processing loop."

pv.py

@@ -13,7 +13,7 @@
 formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 streamHandler.setFormatter(formatter)
 logger.addHandler(streamHandler)
-logger.info("PyFusionSolarDataRelay 1.0.5 started")
+logger.info("PyFusionSolarDataRelay 1.0.6 started")
 
 # Config
 conf = PvConf(logger)

pvconf.py

@@ -39,6 +39,8 @@ def apply_default_settings(self):
         self.gridrelaykenterpasswd = "passwd"
         # Grid infrastructure measurements in The Netherlands, show up in the API with a 3-5 days delay.
         self.gridrelaydaysback = 3
+        # Setting this to 30 would try to backfill gridkenter data on startup for any day between 3 days back (gridrelaydaysback) and 3+30=33 days back.
+        self.gridrelaydaystobackfill = 0
         # If fusionsolar updates every 30mins and meetdata.nl has values per 15min, set this to 2 so that intervals between two datasources match to avoid weird pvoutput graphs.
         self.gridrelaypvoutputspan = 2
 
@@ -79,63 +81,6 @@ def apply_default_settings(self):
         self.mqttpasswd = "fusionsolar"
         self.mqtttopic = "energy/pyfusionsolar"
 
-    def print(self):
-        self.logger.info(f"Current settings:")
-        self.logger.info(f"_Generic:")
-        self.logger.info(f"debug:   {self.debug}")
-        self.logger.info(f"_FusionSolar:")
-        self.logger.info(f"enabled: {self.fusionsolar}")
-        self.logger.info(f"fusionsolarurl: {self.fusionsolarurl}")
-        self.logger.info(f"fusionsolarkkid: {self.fusionsolarkkid}")
-        self.logger.info(f"sysname: {self.pvsysname}")
-        self.logger.info(f"fusionhourcron: {self.fusionhourcron}")
-        self.logger.info(f"fusionminutecron: {self.fusionminutecron}")
-        self.logger.info(f"_Influxdb:")
-        self.logger.info(f"influx: {self.influx}")
-        self.logger.info(f"influx2: {self.influx2}")
-        self.logger.info(f"host: {self.ifhost}")
-        self.logger.info(f"port: {self.ifport}")
-        self.logger.info(f"_Influxdb_v1:")
-        self.logger.info(f"database: {self.if1dbname}")
-        self.logger.info(f"user: {self.if1user}")
-        self.logger.info(f"password: **secret**")
-        self.logger.info(f"_Influxdb_v2:")
-        self.logger.info(f"protocol: {self.if2protocol}")
-        self.logger.info(f"organization: {self.if2org}")
-        self.logger.info(f"bucket: {self.if2bucket}")
-        self.logger.info(f"token: {self.if2token}")
-        self.logger.info(f"_PVOutput.org:")
-        self.logger.info(f"Enabled: {self.pvoutput}")
-        self.logger.info(f"System ID: {self.pvoutputsystemid}")
-        self.logger.info(f"API Key: {self.pvoutputapikey}")
-        self.logger.info(f"API Url: {self.pvoutputurl}")
-        self.logger.info(f"API BatchUrl: {self.pvoutputbatchurl}")
-        self.logger.info(f"_MQTT")
-        self.logger.info(f"Enabled: {self.mqtt}")
-        self.logger.info(f"Host: {self.mqtthost}")
-        self.logger.info(f"Port: {self.mqttport}")
-        self.logger.info(f"Auth: {self.mqttauth}")
-        self.logger.info(f"User: {self.mqttuser}")
-        self.logger.info(f"Passwd: {self.mqttpasswd}")
-        self.logger.info(f"Topic: {self.mqtttopic}")
-        self.logger.info(f"_GridRelay")
-        self.logger.info(f"Enabled: {self.gridrelay}")
-        self.logger.info(f"Interval: {self.gridrelayinterval}")
-        self.logger.info(f"PVOutput span: {self.gridrelaypvoutputspan}")
-        self.logger.info(f"Kenter URL: {self.gridrelaykenterurl}")
-        self.logger.info(f"Days back: {self.gridrelaydaysback}")
-        self.logger.info(f"Kenter User: {self.gridrelaykenteruser}")
-        self.logger.info(f"Kenter Passwd: {self.gridrelaykenterpasswd}")
-
-        self.logger.info(f"System name 01: {self.gridrelaysysname}")
-        self.logger.info(f"Kenter EAN 01: {self.gridrelaykenterean}")
-        self.logger.info(f"Kenter MeterId 01: {self.gridrelaykentermeterid}")
-
-        self.logger.info(f"System name 02: {self.gridrelaysysname02}")
-        self.logger.info(f"Kenter EAN 02: {self.gridrelaykenterean02}")
-        self.logger.info(f"Kenter MeterId: {self.gridrelaykentermeterid02}")
-
-
     def getenv(self, envvar):
         envval = os.getenv(envvar)
         self.logger.debug(f"Pulled '{envvar}={envval}' from the environment")
@@ -154,9 +99,9 @@ def apply_environment_settings(self):
         if os.getenv("pvsysname") != None:
             self.pvsysname = self.getenv("pvsysname")
         if os.getenv("pvfusionhourcron") != None:
-            self.fusionhourcron = int(self.getenv("pvfusionhourcron"))
+            self.fusionhourcron = self.getenv("pvfusionhourcron")
         if os.getenv("pvfusionminutecron") != None:
-            self.fusionminutecron = int(self.getenv("pvfusionminutecron"))
+            self.fusionminutecron = self.getenv("pvfusionminutecron")
         if os.getenv("pvinflux") != None:
             self.influx = self.getenv("pvinflux") == "True"
         if os.getenv("pvinflux2") != None:
@@ -218,6 +163,8 @@ def apply_environment_settings(self):
             self.gridrelaykenterpasswd = self.getenv("pvgridrelaykenterpasswd")
         if os.getenv("pvgridrelaydaysback") != None:
             self.gridrelaydaysback = int(self.getenv("pvgridrelaydaysback"))
+        if os.getenv("pvgridrelaydaystobackfill") != None:
+            self.gridrelaydaystobackfill = int(self.getenv("pvgridrelaydaystobackfill"))
         if os.getenv("pvgridrelaypvoutputspan") != None:
             self.gridrelaypvoutputspan = int(self.getenv("pvgridrelaypvoutputspan"))
 
@@ -237,3 +184,59 @@ def apply_environment_settings(self):
         if os.getenv("pvgridrelaykentermeterid02") != None:
             self.gridrelaykentermeterid02 = self.getenv("pvgridrelaykentermeterid02")
 
+    def print(self):
+        self.logger.info(f"Current settings:")
+        self.logger.info(f"_Generic:")
+        self.logger.info(f"debug:   {self.debug}")
+        self.logger.info(f"_FusionSolar:")
+        self.logger.info(f"enabled: {self.fusionsolar}")
+        self.logger.info(f"fusionsolarurl: {self.fusionsolarurl}")
+        self.logger.info(f"fusionsolarkkid: {self.fusionsolarkkid}")
+        self.logger.info(f"sysname: {self.pvsysname}")
+        self.logger.info(f"fusionhourcron: {self.fusionhourcron}")
+        self.logger.info(f"fusionminutecron: {self.fusionminutecron}")
+        self.logger.info(f"_Influxdb:")
+        self.logger.info(f"influx: {self.influx}")
+        self.logger.info(f"influx2: {self.influx2}")
+        self.logger.info(f"host: {self.ifhost}")
+        self.logger.info(f"port: {self.ifport}")
+        self.logger.info(f"_Influxdb_v1:")
+        self.logger.info(f"database: {self.if1dbname}")
+        self.logger.info(f"user: {self.if1user}")
+        self.logger.info(f"password: **secret**")
+        self.logger.info(f"_Influxdb_v2:")
+        self.logger.info(f"protocol: {self.if2protocol}")
+        self.logger.info(f"organization: {self.if2org}")
+        self.logger.info(f"bucket: {self.if2bucket}")
+        self.logger.info(f"token: {self.if2token}")
+        self.logger.info(f"_PVOutput.org:")
+        self.logger.info(f"Enabled: {self.pvoutput}")
+        self.logger.info(f"System ID: {self.pvoutputsystemid}")
+        self.logger.info(f"API Key: {self.pvoutputapikey}")
+        self.logger.info(f"API Url: {self.pvoutputurl}")
+        self.logger.info(f"API BatchUrl: {self.pvoutputbatchurl}")
+        self.logger.info(f"_MQTT")
+        self.logger.info(f"Enabled: {self.mqtt}")
+        self.logger.info(f"Host: {self.mqtthost}")
+        self.logger.info(f"Port: {self.mqttport}")
+        self.logger.info(f"Auth: {self.mqttauth}")
+        self.logger.info(f"User: {self.mqttuser}")
+        self.logger.info(f"Passwd: {self.mqttpasswd}")
+        self.logger.info(f"Topic: {self.mqtttopic}")
+        self.logger.info(f"_GridRelay")
+        self.logger.info(f"Enabled: {self.gridrelay}")
+        self.logger.info(f"Interval: {self.gridrelayinterval}")
+        self.logger.info(f"PVOutput span: {self.gridrelaypvoutputspan}")
+        self.logger.info(f"Kenter URL: {self.gridrelaykenterurl}")
+        self.logger.info(f"Days back: {self.gridrelaydaysback}")
+        self.logger.info(f"Days to backfill: {self.gridrelaydaystobackfill}")
+        self.logger.info(f"Kenter User: {self.gridrelaykenteruser}")
+        self.logger.info(f"Kenter Passwd: {self.gridrelaykenterpasswd}")
+
+        self.logger.info(f"System name 01: {self.gridrelaysysname}")
+        self.logger.info(f"Kenter EAN 01: {self.gridrelaykenterean}")
+        self.logger.info(f"Kenter MeterId 01: {self.gridrelaykentermeterid}")
+
+        self.logger.info(f"System name 02: {self.gridrelaysysname02}")
+        self.logger.info(f"Kenter EAN 02: {self.gridrelaykenterean02}")
+        self.logger.info(f"Kenter MeterId: {self.gridrelaykentermeterid02}")

pvfusionsolar.py

@@ -22,14 +22,25 @@ def fetch_fusionsolar_status(self):
             )
         except Exception as e:
             raise Exception(
-                "Error fetching data from FusionSolar Kiosk API: '{}'".format(str(e))
+                "Error fetching data from FusionSolar Kiosk API: '{}'".format(
+                    str(e).replace('\n', '')
+                )
             )
 
         try:
             response_json = response.json()
         except Exception as e:
+            content = response.content.decode("utf-8")
+            if content:
+                content_first_100 = content[:200]
+            else:
+                content_first_100 = ""
+                
             raise Exception(
-                "Error while parsing JSON response from Kiosk API: '{}'".format(str(e))
+                "Error while decoding the JSON Kiosk API response, did you set the right fusionsolarurl and fusionsolarkkid in your conf? Kiosk link still working?: '{}', raw JSON content: '{}'".format(
+                    str(e).replace('\n', ''),
+                    content_first_100.replace('\n', '')
+                )
             )
 
         if not "data" in response_json:
@@ -42,8 +53,9 @@ def fetch_fusionsolar_status(self):
             response_json_data = json.loads(response_json_data_decoded)
         except Exception as e:
             raise Exception(
-                "Error while parsing JSON response data element from FusionSolar Kiosk API: '{}'".format(
-                    str(e)
+                "Error while parsing JSON response data element from FusionSolar Kiosk API: '{}' Data element content from FusionSolar API: {}".format(
+                    str(e).replace('\n', ''),
+                    response_json["data"]
                 )
             )
 

pvrelay.py

@@ -25,6 +25,10 @@ def __init__(self, conf: PvConf, logger):
         self.logger.debug("PvRelay waiting 5sec to initialize docker-compose containers")
         time.sleep(5)
 
+        if self.conf.debug:
+            self.logger.info("Starting process_fusionsolar_request() at init, before waiting for cron, because we're in debug mode")
+            self.process_fusionsolar_request()
+
         sched = BlockingScheduler(standalone = True)
         sched.add_job(self.process_fusionsolar_request, trigger='cron', hour=self.conf.fusionhourcron, minute=self.conf.fusionminutecron)
         sched.start()