jsonread plugin: make it work without pyjq under python 3.13 (limited functionality)

Author: onkelandy

jsonread/__init__.py

@@ -27,119 +27,228 @@
 import json
 import requests
 from requests_file import FileAdapter
-import pyjq
+import re
 from lib.model.smartplugin import SmartPlugin
 from lib.item import Items
 from .webif import WebInterface
 
+def jq_compile(expr):
+    """Split filter expression into pipe steps"""
+    return tuple(p.strip() for p in expr.strip().split("|"))
+
+def jq_full(pipes, value):
+    """Apply pipe chain like jq"""
+    for pipe in pipes:
+        pipe = pipe.strip()
+        if not isinstance(value, list):
+            value = [value]
+        out = []
+        for v in value:
+            res = jq_step(pipe, v)
+            if isinstance(res, list):
+                out.extend(res)
+            elif res is not None:
+                out.append(res)
+        value = out
+    return value
+
+def jq_step(expr, value):
+    expr = expr.strip()
+
+    # select() über Listen
+    if expr.startswith("select(") and expr.endswith(")"):
+        cond = expr[7:-1]
+        if isinstance(value, list):
+            return [v for v in value if jq_condition(cond, v)]
+        return [value] if jq_condition(cond, value) else []
+
+    # [] Operator
+    if expr.endswith("[]"):
+        base = expr[:-2]
+        res = jq_path(base, value)
+        if isinstance(res, list):
+            return res
+        return []
+
+    # normal path
+    return jq_path(expr, value)
+
+def jq_condition(cond, obj):
+    m = re.match(r'\.(.+?)\s*==\s*("?)(.*?)\2$', cond)
+    if m:
+        key, _, val = m.groups()
+        if isinstance(obj, dict):
+            return str(obj.get(key)) == val
+    return False
+
+def jq_path(path, data):
+    """Resolve a dot-separated path, handling [] and quoted keys"""
+    path = path.lstrip(".")
+    if path == "":
+        return data
+
+    # Split respecting quoted keys
+    parts = []
+    buf = ""
+    in_quotes = False
+    for ch in path:
+        if ch == '"':
+            in_quotes = not in_quotes
+            buf += ch
+        elif ch == "." and not in_quotes:
+            parts.append(buf)
+            buf = ""
+        else:
+            buf += ch
+    if buf:
+        parts.append(buf)
+
+    def normalize_key(k):
+        k = k.strip()
+        if k.startswith('"') and k.endswith('"'):
+            return k[1:-1]
+        return k
+
+    vals = [data]
+    for part in parts:
+        key = normalize_key(part)
+        is_list = False
+        if key.endswith("[]"):
+            key = key[:-2]
+            is_list = True
+        new_vals = []
+        for v in vals:
+            if isinstance(v, dict) and key in v:
+                val = v[key]
+                if is_list:
+                    if isinstance(val, list):
+                        new_vals.extend(val)
+                    else:
+                        new_vals.append(val)
+                else:
+                    new_vals.append(val)
+            elif isinstance(v, list):
+                for e in v:
+                    if isinstance(e, dict) and key in e:
+                        val = e[key]
+                        if is_list and isinstance(val, list):
+                            new_vals.extend(val)
+                        else:
+                            new_vals.append(val)
+        vals = new_vals
+    # pyjq.first() compatibility
+    if len(vals) == 0:
+        return None
+    if len(vals) == 1:
+        return vals[0]
+    return vals
+
+def jq_unwrap(value):
+    """pyjq.first()-compatibility"""
+    if isinstance(value, list):
+        if len(value) == 0:
+            return None
+        if len(value) == 1:
+            return value[0]
+    return value
+
+# ============================================================
+# JSONREAD Plugin (Turbo + jq kompatibel)
+# ============================================================
 
 class JSONREAD(SmartPlugin):
-    PLUGIN_VERSION = "1.0.4"
+    PLUGIN_VERSION = "2.0.0"
 
     def __init__(self, sh):
-        """
-        Initializes the plugin
-        @param url: URL of the json data to fetch
-        @param cycle: the polling interval in seconds
-        """
-        # Call init code of parent class (SmartPlugin)
         super().__init__()
 
-        from bin.smarthome import VERSION
-        if '.'.join(VERSION.split('.', 2)[:2]) <= '1.5':
-            self.logger = logging.getLogger(__name__)
-
         self._url = self.get_parameter_value('url')
         self._cycle = self.get_parameter_value('cycle')
+
         self._session = requests.Session()
         self._session.mount('file://', FileAdapter())
+
         self._items = {}
+        self._compiled_filters = {}
+
         self._lastresult = {}
         self._lastresultstr = ""
         self._lastresultjq = ""
 
-        # if plugin should start even without web interface
         self.init_webinterface(WebInterface)
 
-
     def run(self):
-        """
-        Run method for the plugin
-        """
         self.logger.debug("Run method called")
         self.alive = True
         self.scheduler_add(self.get_fullname(), self.poll_device, cycle=self._cycle)
 
     def stop(self):
         self.logger.debug("Stop method called")
-        self.scheduler_remove(self.get_fullname() )
+        self.scheduler_remove(self.get_fullname())
         self.alive = False
 
     def parse_item(self, item):
         if self.has_iattr(item.conf, 'jsonread_filter'):
-            self._items[item] = self.get_iattr_value(item.conf, 'jsonread_filter')
+            expr = self.get_iattr_value(item.conf, 'jsonread_filter')
+            self._items[item] = expr
+            self._compiled_filters[item] = jq_compile(expr)
 
     def poll_device(self):
         try:
             response = self._session.get(self._url)
-
         except Exception as ex:
-            self.logger.error("Exception when sending GET request for {}: {}".format(self._url,str(ex)))
+            self.logger.error(f"GET failed {self._url}: {ex}")
             return
 
         if response.status_code != 200:
-            self.logger.error("Bad response code from GET '{}': {}".format(self._url, response.status_code))
+            self.logger.error(f"Bad HTTP {response.status_code} from {self._url}")
             return
 
         try:
             json_obj = response.json()
-        except Exception as ex:
-            self.logger.error("Response from '{}' doesn't look like json '{}'".format(self._url, str(response.content)[:30]))
+        except Exception:
+            self.logger.error(f"Response from {self._url} is not JSON")
             return
 
+        # Store debug info (Unicode-safe)
         try:
             self._lastresult = json_obj
-            self._lastresultstr = json.dumps(self._lastresult, indent=4, sort_keys=True)
-            self._lastresultjq = '\n'.join(str(x) for x in pathes(self._lastresult))
-        except Exception as ex:
-            self.logger.error("Could not change '{}' into pretty json string'{}'".format(self._lastresult,self._lastresultstr))
-            self._lastresultstr = "<empty due to failure>"
+            self._lastresultstr = json.dumps(json_obj, indent=4, sort_keys=True, ensure_ascii=False)
+            self._lastresultjq = '\n'.join(pathes(json_obj))
+        except Exception:
+            self._lastresultstr = "<format error>"
 
-        for k in self._items.keys():
+        # Process items (Turbo mode)
+        for item, expr in self._items.items():
             try:
-                jqres = pyjq.first(self._items[k], json_obj)
-
+                compiled = self._compiled_filters[item]
+                jqres = jq_full(compiled, json_obj)
+                jqres = jq_unwrap(jqres)
             except Exception as ex:
-                self.logger.error("jq filter failed: {}'".format(str(ex)))
+                self.logger.error(f"jq failed: {expr} => {ex}")
                 continue
 
-            k(jqres)
+            try:
+                item(jqres)
+            except Exception as ex:
+                self.logger.error(f"Item update failed {item}: {ex}")
 
-# just a helper function
+# ============================================================
+# Debug helper
+# ============================================================
 
-def pathes( d, stem=""):
-    #print("Stem:",stem)
+def pathes(d, stem=""):
     if isinstance(d, dict):
         for key, value in d.items():
-            if isinstance(value, dict):
-                for d in pathes(value, "{}.{}".format(stem,key)):
-                    yield d
-            elif isinstance(value, list) or isinstance(value, tuple):
-                for v in value:
-                    for d in pathes(v, "{}.{}".format(stem,key)):
-                        yield d
+            if isinstance(value, (dict, list, tuple)):
+                yield from pathes(value, f"{stem}.{key}")
             else:
-                yield "{}.{} => {}".format(stem,key,value)
-    elif isinstance(d, list) or isinstance(d, tuple):
+                yield f"{stem}.{key} => {value}"
+    elif isinstance(d, (list, tuple)):
         for value in d:
-            if isinstance(value, dict):
-                for d in pathes(value, "{}.{}".format(stem,key)):
-                    yield d
-            elif isinstance(value, list) or isinstance(value, tuple):
-                for v in value:
-                    for d in pathes(v, "{}.{}".format(stem,key)):
-                        yield d
+            if isinstance(value, (dict, list, tuple)):
+                yield from pathes(value, stem)
             else:
-                yield "{}.{} => {}".format(stem,key,value)
+                yield f"{stem} => {value}"
     else:
-        yield "{}.{}".format(stem,d)
+        yield f"{stem}.{d}"

jsonread/plugin.yaml

@@ -3,21 +3,19 @@ plugin:
     # Global plugin attributes
     type: web               # plugin type (gateway, interface, protocol, system, web)
     description:
-        de: 'json Parser Plugin basierend auf jq'
-        en: 'json parser plugin based on jq'
+        de: 'json Parser Plugin ohne weitere Module'
+        en: 'json parser plugin ohne weitere Module'
     maintainer: onkelandy
     tester: bmxp
     state: ready
-    keywords: json jq
-    documentation: http://smarthomeng.de/user/plugins_doc/config/not-yet.html
+    keywords: json
+    documentation: http://smarthomeng.de/user/plugins_doc/config/jsonread.html
     support: https://knx-user-forum.de/forum/supportforen/smarthome-py/not-yet
 
-    version: 1.0.4                 # Plugin version
+    version: 2.0.0                 # Plugin version
+
+    py_minversion: 3.9            # minimum Python version to use for this plugin
     sh_minversion: '1.4'             # minimum shNG version to use this plugin
-    #sh_maxversion:                # maximum shNG version to use this plugin (leave empty if latest)
-    #py_minversion: 3.6            # minimum Python version to use for this plugin
-    py_maxversion: '3.11'            # maximum Python version to use for this plugin (leave empty if latest)
-    py_versioncomment: "Die aktuellste Version (2.6.0, released August 2022) des benötigten Packages 'pyjq' ist nicht kompatibel mit Python 3.12"
     restartable: True
     multi_instance: True           # plugin supports multi instance
     classname: JSONREAD            # class containing the plugin
@@ -42,22 +40,14 @@ parameters:
 
 
 item_attributes:
-    # Definition of item attributes defined by this plugin
     jsonread_filter:
         type: str
         description:
             de: 'JQ Pfad um innerhalb eines JSON Datensatzes einen Wert auszuwählen. Dieser Wert wird dem Item dann zugewiesen'
             en: 'JQ path to select a value from JSON dataset. The Item will then receive this value'
 
 item_structs: NONE
-    # Definition of item-structure templates for this plugin (enter 'item_structs: NONE', if section should be empty)
-
-#item_attribute_prefixes:
-    # Definition of item attributes that only have a common prefix (enter 'item_attribute_prefixes: NONE' or ommit this section, if section should be empty)
-    # NOTE: This section should only be used, if really nessesary (e.g. for the stateengine plugin)
 
 logic_parameters: NONE
-    # Definition of logic parameters defined by this plugin (enter 'logic_parameters: NONE', if section should be empty)
 
 plugin_functions: NONE
-    # Definition of function interface of the plugin

jsonread/requirements.txt

@@ -1,8 +1 @@
-#requests requirement moved to core
 requests-file
-# pyjq:
-#   The most recent version (2.6 released August 2022) is compatible with Python 3.10
-pyjq
-#   The version (2.5.2 released May 2021) is not compatible with Python 3.10
-#   The project's README states, that the project is deprecated in favor Slixmpp (a fork of sleekxmpp).
-#pyjq;python_version<'3.10'