Merge branch 'main' into parallel_fix

Author: jaredmauch

classes/protocol_settings.py

@@ -338,6 +338,19 @@ def get_registry_entry(self, name : str, registry_type : Registry_Type) -> regis
                 return item
 
         return None
+    
+    def get_code_by_value(self, entry : registry_map_entry, value : str, fallback=None) -> str:
+        ''' case insensitive '''
+        
+        value = value.strip().lower()
+
+        if entry.variable_name+"_codes" in self.codes:
+            codes = self.codes[entry.variable_name+"_codes"]
+            for code, val in codes.items():
+                if value == val.lower():
+                    return code
+                
+        return fallback
 
     def load__json(self, file : str = "", settings_dir : str = ""):
         if not settings_dir:

classes/transports/influxdb_out.py

@@ -1,12 +1,12 @@
-import sys
+import logging
 import os
-import json
 import pickle
-from configparser import SectionProxy
-from typing import TextIO
 import time
 import logging
 import threading
+from configparser import SectionProxy
+
+from influxdb import InfluxDBClient
 
 from defs.common import strtobool
 
@@ -209,7 +209,6 @@ def connect(self):
         self._log.info("influxdb_out connect")
 
         try:
-            from influxdb import InfluxDBClient
 
             # Create InfluxDB client with timeout settings
             self.client = InfluxDBClient(

classes/transports/modbus_base.py

@@ -810,7 +810,7 @@ def evaluate_score(entry : registry_map_entry, val):
     def write_variable(self, entry : registry_map_entry, value : str, registry_type : Registry_Type = Registry_Type.HOLDING):
         """ writes a value to a ModBus register; todo: registry_type to handle other write functions"""
 
-        value = value.strip()
+        value = value.strip().lower()
 
         temp_map = [entry]
         ranges = self.protocolSettings.calculate_registry_ranges(temp_map, self.protocolSettings.registry_map_size[registry_type], init=True) #init=True to bypass timechecks
@@ -820,6 +820,11 @@ def write_variable(self, entry : registry_map_entry, value : str, registry_type
         #current_registers = self.read_modbus_registers(start=entry.register, end=entry.register, registry_type=registry_type)
         #current_value = current_registers[entry.register]
         current_value = info[entry.variable_name]
+        
+
+        #handle codes
+        value = self.protocolSettings.get_code_by_value(entry, value, fallback=value)
+        current_value = self.protocolSettings.get_code_by_value(entry, current_value, fallback=current_value)
 
         if not self.write_mode == TransportWriteMode.UNSAFE:
             if not self.protocolSettings.validate_registry_entry(entry, current_value):
@@ -830,14 +835,6 @@ def write_variable(self, entry : registry_map_entry, value : str, registry_type
                 if not self.protocolSettings.validate_registry_entry(entry, value):
                     return self._log.error(f"WRITE_ERROR: Invalid new value, '{value}'. Unsafe to write")
 
-        #handle codes
-        if entry.variable_name+"_codes" in self.protocolSettings.codes:
-            codes = self.protocolSettings.codes[entry.variable_name+"_codes"]
-            for key, val in codes.items():
-                if val == value: #convert "string" to key value
-                    value = key
-                    break
-
         #apply unit_mod before writing.
         if entry.unit_mod != 1:
             value = int(float(value) / entry.unit_mod) # say unitmod is 0.1. 105*0.1 = 10.5. 10.5 / 0.1 = 105.

classes/transports/transport_base.py

@@ -112,7 +112,7 @@ def __init__(self, settings : "SectionProxy") -> None:
 
             #load a protocol_settings class for every transport; required for adv features. ie, variable timing.
             #must load after settings
-            self.protocol_version = settings.get("protocol_version")
+            self.protocol_version = settings.get("protocol_version", fallback='')
             if self.protocol_version:
                 # Create a deep copy of protocol settings to avoid shared state between transports
                 original_protocol_settings = protocol_settings(self.protocol_version, transport_settings=settings)

documentation/usage/devices_and_protocols.csv

@@ -0,0 +1,16 @@
+Brand,Model,Protocol,Transport,ReadMe
+AOLithium,51.2V 100Ah,voltronic_bms_2020_03_25,ModBus,https://github.com/HotNoob/PythonProtocolGateway/blob/main/documentation/devices/AOLithium.md
+AOLithium,51.2V 100Ah,victron_gx_generic_canbus,CanBus,https://github.com/HotNoob/PythonProtocolGateway/blob/main/documentation/devices/AOLithium.md
+EG4,EG4 18kPV,eg4_v58,ModBus,
+EG4,EG4 3000EHV ,eg4_3000ehv_v1 ,ModBus,
+EG4,EG4 6000XP ,eg4_v58,ModBus,
+EG4,MPPT100-48HV – unconfirmed,eg4_3000ehv_v1 ,ModBus,
+Growatt,SPF 12000T DVM-US MPV,v0.14,ModBus,
+Growatt,SPF 5000 ,v0.14,ModBus,
+Growatt,SPF 6000 ,v0.14,ModBus,
+Selphos,v3,growatt_bms_canbus_v1.04 ,CanBus,https://github.com/HotNoob/PythonProtocolGateway/discussions/88
+Sigineer,M3000H-48LV-3KW ,sigineer_v0.11,ModBus,https://github.com/HotNoob/PythonProtocolGateway/blob/main/documentation/devices/Sigineer.md
+SOK,48v100ah,pace_bms_v1.3 ,ModBus,https://github.com/HotNoob/PythonProtocolGateway/blob/main/documentation/devices/SOK.md
+SolArk,Untested,solark_v1.1 ,ModBus,
+SRNE,ASF48100S200-H ,srne_v3.9,ModBus,
+SRNE,HF2430U60-100 ,srne_v3.9,ModBus,

protocol_gateway.py

@@ -28,6 +28,7 @@
 
 from classes.protocol_settings import protocol_settings, registry_map_entry
 from classes.transports.transport_base import transport_base
+from defs.common import strtobool
 
 __logo = """
 
@@ -50,13 +51,13 @@
 
 class CustomConfigParser(ConfigParser):
     def get(self, section, option, *args, **kwargs):
-        if isinstance(option, list):
-            fallback = None
+        fallback = None
 
-            if "fallback" in kwargs: #override kwargs fallback, for manually handling here
-                fallback = kwargs["fallback"]
-                kwargs["fallback"] = None
+        if "fallback" in kwargs: #override kwargs fallback, for manually handling here
+            fallback = kwargs["fallback"]
+            kwargs["fallback"] = None
 
+        if isinstance(option, list):
             for name in option:
                 try:
                     value = super().get(section, name, *args, **kwargs)
@@ -65,14 +66,20 @@ def get(self, section, option, *args, **kwargs):
 
                 if value:
                     break
+        else:
+            try:
+                value = super().get(section, option, *args, **kwargs)
+            except NoOptionError:
+                value = None
 
-            if not value:
-                value = fallback
+        if not value: #apply fallback
+            value = fallback
 
-            if value is None:
+        if value is None:
+            if isinstance(option, list):
                 raise NoOptionError(option[0], section)
-        else:
-            value = super().get(section, option, *args, **kwargs)
+            else:
+                raise NoOptionError(option, section)
 
         if isinstance(value, int):
             return value
@@ -95,6 +102,10 @@ def getint(self, section, option, *args, **kwargs): #bypass fallback bug
     def getfloat(self, section, option, *args, **kwargs): #bypass fallback bug
         value = self.get(section, option, *args, **kwargs)
         return float(value) if value is not None else None
+    
+    def getboolean(self, section, option, *args, **kwargs): #bypass fallback bug
+        value = self.get(section, option, *args, **kwargs)
+        return strtobool(value)
 
     def getboolean(self, section, option, *args, **kwargs): #bypass fallback bug and handle case-insensitive boolean values
         value = self.get(section, option, *args, **kwargs)

protocols/srne/srne_2021_v1.96.holding_registry_map.csv

@@ -45,23 +45,44 @@ variable name,data type,register,documented name,description,writable,values,uni
 ,,0x0233,Load Phase_C active power,,R,,W,
 ,,0x0235,Load Phase_C apparent power,,R,,VA,
 ,,0x0237,Load Phase_C ratio,,R,0~100,%,
-,BYTE,0xF000,Stats PVEnergyYesterday,,R,,0.1kWh,
-,BYTE,0xF001,Stats PVEnergy2Dayago,,R,,0.1kWh,
-,BYTE,0xF002,Stats PVEnergy3Dayago,,R,,0.1kWh,
-,BYTE,0xF003,Stats PVEnergy4Dayago,,R,,0.1kWh,
-,BYTE,0xF004,Stats PVEnergy5Dayago,,R,,0.1kWh,
-,BYTE,0xF005,Stats PVEnergy6Dayago,,R,,0.1kWh,
-,BYTE,0xF006,Stats PVEnergy7Dayago,,R,,0.1kWh,
-,BYTE,0xF02C,Stats GenerateEnergyToGridTday,,R,,0.1kWh,
-,BYTE,0xF02D,Stats BatChgTday,,R,,1AH,
-,BYTE,0xF02E,Stats BatDischgTday,,R,,1AH,
-,BYTE,0xF02F,Stats GenerateEnergyTday,,R,,0.1kWh,
-,BYTE,0xF030,Stats UsedEnergyTday,,R,,0.1kWh,
-,BYTE,0xF031,Stats WorkDaysTotal,,R,,1d,
-,BYTE,0xF038,Stats GeneratEnergyTotal,,R,,0.1kWh,
-,BYTE,0xF03C,Stats GridChgEnergyTday,,R,,1AH,
-,BYTE,0xF03D,Stats LoadConsumLineTday,,R,,0.1kWh,
-,BYTE,0xF03E,Stats InvWorkTimeTday,,R,,1min,
-,BYTE,0xF03F,Stats GridWorkTimeTday,,R,,1min,
-,BYTE,0xF04A,Stats InvWorkTimeTotal,,R,,1h,
-,BYTE,0xF04B,Stats GridWorkTimeTotal,,R,,1h,
+,,0xF000,Stats PVEnergyYesterday,,R,,0.1kWh,
+,,0xF001,Stats PVEnergy2Dayago,,R,,0.1kWh,
+,,0xF002,Stats PVEnergy3Dayago,,R,,0.1kWh,
+,,0xF003,Stats PVEnergy4Dayago,,R,,0.1kWh,
+,,0xF004,Stats PVEnergy5Dayago,,R,,0.1kWh,
+,,0xF005,Stats PVEnergy6Dayago,,R,,0.1kWh,
+,,0xF006,Stats PVEnergy7Dayago,,R,,0.1kWh,
+,,0xF007,Stats BatChgEnergyYesterday,,R,,1AH,
+,,0xF008,Stats BatChgEnergy2Dayago,,R,,1AH,
+,,0xF009,Stats BatChgEnergy3Dayago,,R,,1AH,
+,,0xF00A,Stats BatChgEnergy4Dayago,,R,,1AH,
+,,0xF00B,Stats BatChgEnergy5Dayago,,R,,1AH,
+,,0xF00C,Stats BatChgEnergy6Dayago,,R,,1AH,
+,,0xF00D,Stats BatChgEnergy7Dayago,,R,,1AH,
+,,0xF00E,Stats BatDischgEnergyYesterday,,R,,1AH,
+,,0xF00F,Stats BatDischgEnergy2Dayago,,R,,1AH,
+,,0xF010,Stats BatDischgEnergy3Dayago,,R,,1AH,
+,,0xF011,Stats BatDischgEnergy4Dayago,,R,,1AH,
+,,0xF012,Stats BatDischgEnergy5Dayago,,R,,1AH,
+,,0xF013,Stats BatDischgEnergy6Dayago,,R,,1AH,
+,,0xF014,Stats BatDischgEnergy7Dayago,,R,,1AH,
+,,0xF015,Stats GridChgEnergyYesterday,,R,,1AH,
+,,0xF016,Stats GridChgEnergy2Dayago,,R,,1AH,
+,,0xF017,Stats GridChgEnergy3Dayago,,R,,1AH,
+,,0xF018,Stats GridChgEnergy4Dayago,,R,,1AH,
+,,0xF019,Stats GridChgEnergy5Dayago,,R,,1AH,
+,,0xF01A,Stats GridChgEnergy6Dayago,,R,,1AH,
+,,0xF01B,Stats GridChgEnergy7Dayago,,R,,1AH,
+,,0xF02C,Stats GenerateEnergyToGridTday,,R,,0.1kWh,
+,,0xF02D,Stats BatChgTday,,R,,1AH,
+,,0xF02E,Stats BatDischgTday,,R,,1AH,
+,,0xF02F,Stats GenerateEnergyTday,,R,,0.1kWh,
+,,0xF030,Stats UsedEnergyTday,,R,,0.1kWh,
+,,0xF031,Stats WorkDaysTotal,,R,,1d,
+,,0xF038,Stats GeneratEnergyTotal,,R,,0.1kWh,
+,,0xF03C,Stats GridChgEnergyTday,,R,,1AH,
+,,0xF03D,Stats GridLoadConsumTday,,R,,0.1kWh,
+,,0xF03E,Stats InvWorkTimeTday,,R,,1min,
+,,0xF03F,Stats GridWorkTimeTday,,R,,1min,
+,,0xF04A,Stats InvWorkTimeTotal,,R,,1h,
+,,0xF04B,Stats GridWorkTimeTotal,,R,,1h,

pytests/test_influxdb_out.py

@@ -3,9 +3,10 @@
 Test for InfluxDB output transport
 """
 
+import time
 import unittest
-from unittest.mock import Mock, patch, MagicMock
-from configparser import ConfigParser
+from protocol_gateway import CustomConfigParser as ConfigParser
+from unittest.mock import MagicMock, Mock, patch
 
 from classes.transports.influxdb_out import influxdb_out
 
@@ -22,24 +23,27 @@ def setUp(self):
         self.config.set('influxdb_output', 'port', '8086')
         self.config.set('influxdb_output', 'database', 'test_db')
 
+    #@patch('classes.transports.influxdb_out.InfluxDBClient')
+    #@patch('classes.transports.influxdb_out.InfluxDBClient')
     @patch('classes.transports.influxdb_out.InfluxDBClient')
     def test_connect_success(self, mock_influxdb_client):
         """Test successful connection to InfluxDB"""
         # Mock the InfluxDB client
         mock_client = Mock()
         mock_influxdb_client.return_value = mock_client
         mock_client.get_list_database.return_value = [{'name': 'test_db'}]
-        
+
         transport = influxdb_out(self.config['influxdb_output'])
         transport.connect()
-        
+
         self.assertTrue(transport.connected)
         mock_influxdb_client.assert_called_once_with(
             host='localhost',
             port=8086,
             username=None,
             password=None,
-            database='test_db'
+            database='test_db',
+            timeout=10
         )
 
     @patch('classes.transports.influxdb_out.InfluxDBClient')
@@ -49,10 +53,10 @@ def test_connect_database_creation(self, mock_influxdb_client):
         mock_client = Mock()
         mock_influxdb_client.return_value = mock_client
         mock_client.get_list_database.return_value = [{'name': 'other_db'}]
-        
+
         transport = influxdb_out(self.config['influxdb_output'])
         transport.connect()
-        
+
         self.assertTrue(transport.connected)
         mock_client.create_database.assert_called_once_with('test_db')
 
@@ -63,10 +67,11 @@ def test_write_data_batching(self, mock_influxdb_client):
         mock_client = Mock()
         mock_influxdb_client.return_value = mock_client
         mock_client.get_list_database.return_value = [{'name': 'test_db'}]
-        
+
         transport = influxdb_out(self.config['influxdb_output'])
         transport.connect()
-        
+
+
         # Mock source transport
         source_transport = Mock()
         source_transport.transport_name = 'test_source'
@@ -75,21 +80,27 @@ def test_write_data_batching(self, mock_influxdb_client):
         source_transport.device_manufacturer = 'Test Manufacturer'
         source_transport.device_model = 'Test Model'
         source_transport.device_serial_number = '123456'
-        
+
+        mock_protocol_settings = Mock()
+        mock_protocol_settings.get_registry_map.return_value = []  # or list of entries if you want
+        source_transport.protocolSettings = mock_protocol_settings
+
         # Test data
         test_data = {'battery_voltage': '48.5', 'battery_current': '10.2'}
-        
+
+        transport.last_batch_time = time.time() #stop "flush" from happening and failing test
+        transport.batch_timeout = 21
         transport.write_data(test_data, source_transport)
-        
+
         # Check that data was added to batch
         self.assertEqual(len(transport.batch_points), 1)
         point = transport.batch_points[0]
-        
+
         self.assertEqual(point['measurement'], 'device_data')
         self.assertIn('device_identifier', point['tags'])
         self.assertIn('battery_voltage', point['fields'])
         self.assertIn('battery_current', point['fields'])
-        
+
         # Check data type conversion
         self.assertEqual(point['fields']['battery_voltage'], 48.5)
         self.assertEqual(point['fields']['battery_current'], 10.2)
@@ -102,9 +113,9 @@ def test_configuration_options(self):
         self.config.set('influxdb_output', 'measurement', 'custom_measurement')
         self.config.set('influxdb_output', 'batch_size', '50')
         self.config.set('influxdb_output', 'batch_timeout', '5.0')
-        
+
         transport = influxdb_out(self.config['influxdb_output'])
-        
+
         self.assertEqual(transport.username, 'admin')
         self.assertEqual(transport.password, 'secret')
         self.assertEqual(transport.measurement, 'custom_measurement')