Migrate ophyd async 0.10 (#33)

* set new typevar

* added 3.12 to testing

* add bson dependenc

* tests run

* migrate to v0.10

* test fix

---------

Co-authored-by: Peter Braun <peter.braun@helmholtz-berlin.de>

Author: Bilchreis

.github/workflows/code.yaml

@@ -43,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4

pyproject.toml

@@ -10,7 +10,7 @@ dynamic = ["version"]
 description = "An Interface between bluesky and SECoP, using ophyd and frappy-client"
 
 dependencies = [
-    'ophyd-async >= 0.9.0',
+    'ophyd-async >= 0.10.0',
     'frappy-core == 0.20.4'
 
 ]
@@ -37,7 +37,7 @@ dev = [
     'pytest-xprocess',
     'pre-commit',
     'ipykernel',
-    'databroker',
+    'databroker[all]==2.0.0b64',
     'ophyd',
     'tox',
     'mlzlog',
@@ -60,7 +60,7 @@ minversion = "6.0"
 addopts = "-v"
 asyncio_mode="auto"
 filterwarnings = "ignore::PendingDeprecationWarning"
-norecursedirs=["frappy","epics-testing","build"]
+
 
 [tool.isort]
 float_to_top = true

src/secop_ophyd/SECoPDevices.py

@@ -10,15 +10,13 @@
 
 import bluesky.plan_stubs as bps
 from bluesky.protocols import (
-    Descriptor,
     Flyable,
     Locatable,
     Location,
     PartialEvent,
     Reading,
     Stoppable,
     Subscribable,
-    SyncOrAsync,
     Triggerable,
 )
 from frappy.datatypes import (
@@ -222,9 +220,6 @@ def collect(self) -> Iterator[PartialEvent]:
             time=self._start_time, timestamps={self.name: []}, data={self.name: []}
         )
 
-    async def describe_collect(self) -> SyncOrAsync[Dict[str, Dict[str, Descriptor]]]:
-        return await self.describe()
-
 
 class SECoPBaseDevice(StandardReadable):
     """Base Class for generating Opyd devices from SEC Node modules,

src/secop_ophyd/SECoPSignal.py

@@ -1,7 +1,7 @@
 import asyncio
 import warnings
 from functools import wraps
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict
 
 from bluesky.protocols import DataKey, Reading
 from frappy.client import CacheItem
@@ -17,10 +17,10 @@
     StructOf,
     TupleOf,
 )
-from ophyd_async.core import Callback, SignalBackend, T
+from ophyd_async.core import Callback, SignalBackend, SignalDatatypeT
 
 from secop_ophyd.AsyncFrappyClient import AsyncFrappyClient
-from secop_ophyd.util import Path, SECoPdtype, SECoPReading, deep_get
+from secop_ophyd.util import Path, SECoPDataKey, SECoPdtype, SECoPReading, deep_get
 
 atomic_dtypes = (
     StringType,
@@ -105,18 +105,18 @@ async def put(self, value: Any | None, wait=True):
 
     async def get_datakey(self, source: str) -> DataKey:
         """Metadata like source, dtype, shape, precision, units"""
-        return self.describe_dict
+        return describedict_to_datakey(self.describe_dict)
 
-    async def get_reading(self) -> Reading:
+    async def get_reading(self) -> Reading[SignalDatatypeT]:
         return self.reading.get_reading()
 
-    async def get_value(self) -> T:
+    async def get_value(self) -> SignalDatatypeT:
         return self.reading.get_value()
 
-    async def get_setpoint(self) -> T:
+    async def get_setpoint(self) -> SignalDatatypeT:
         return await self.get_value()
 
-    def set_callback(self, callback: Callback[T] | None) -> None:
+    def set_callback(self, callback: Callback[Reading[SignalDatatypeT]] | None) -> None:
         self.callback = callback  # type: ignore[assignment]
 
 
@@ -191,36 +191,25 @@ async def put(self, value: Any | None, wait=True):
 
     async def get_datakey(self, source: str) -> DataKey:
         """Metadata like source, dtype, shape, precision, units"""
-        res = {}
 
-        res["source"] = self.source("", True)
+        return DataKey(shape=[], dtype="string", source=self.source("", True))
 
-        # ophyd datatype (some SECoP datatypeshaveto be converted)
-        # signalx has no datatype and is never read
-        res["dtype"] = "None"
-
-        # get shape from datainfo and SECoPtype
-
-        res["shape"] = []  # type: ignore
-
-        return res
-
-    async def get_reading(self) -> Reading:
+    async def get_reading(self) -> Reading[SignalDatatypeT]:
         raise Exception(
             "Cannot read _x Signal, it has no value and is only"
             + " used to trigger Command execution"
         )
 
-    async def get_value(self) -> T:
+    async def get_value(self) -> SignalDatatypeT:
         raise Exception(
             "Cannot read _x Signal, it has no value and is only"
             + " used to trigger Command execution"
         )
 
-    def set_callback(self, callback: Callback[T] | None) -> None:
+    def set_callback(self, callback: Callback[Reading[SignalDatatypeT]] | None) -> None:
         pass
 
-    async def get_setpoint(self) -> T:
+    async def get_setpoint(self) -> SignalDatatypeT:
         raise Exception(
             "Cannot read _x Signal, it has no value and is only"
             + " used to trigger Command execution"
@@ -334,9 +323,9 @@ async def get_datakey(self, source: str) -> DataKey:
             SECoPReading(entry=dataset, secop_dt=self.SECoP_type_info)
             self.describe_dict.update(self.SECoP_type_info.get_datakey())
 
-        return self.describe_dict
+        return describedict_to_datakey(self.describe_dict)
 
-    async def get_reading(self) -> Reading:
+    async def get_reading(self) -> Reading[SignalDatatypeT]:
         dataset = await self._secclient.get_parameter(
             **self.get_param_path(), trycache=False
         )
@@ -345,15 +334,15 @@ async def get_reading(self) -> Reading:
 
         return sec_reading.get_reading()
 
-    async def get_value(self) -> T:
+    async def get_value(self) -> SignalDatatypeT:
         dataset: Reading = await self.get_reading()
 
         return dataset["value"]  # type: ignore
 
-    async def get_setpoint(self) -> T:
+    async def get_setpoint(self) -> SignalDatatypeT:
         return await self.get_value()
 
-    def set_callback(self, callback: Callback[T] | None) -> None:
+    def set_callback(self, callback: Callback[Reading[SignalDatatypeT]] | None) -> None:
         def awaitify(sync_func):
             """Wrap a synchronous callable to allow ``await``'ing it"""
 
@@ -403,7 +392,7 @@ class PropertyBackend(SignalBackend):
     """Readonly backend for static SECoP Properties of Nodes/Modules"""
 
     def __init__(
-        self, prop_key: str, property_dict: Dict[str, T], secclient: AsyncFrappyClient
+        self, prop_key: str, property_dict: Dict[str, Any], secclient: AsyncFrappyClient
     ) -> None:
         """Initializes PropertyBackend
 
@@ -450,16 +439,16 @@ async def connect(self, timeout: float):
         """Connect to underlying hardware"""
         pass
 
-    async def put(self, value: Optional[T], wait=True):
+    async def put(self, value: SignalDatatypeT | None, wait=True):
         """Put a value to the PV, if wait then wait for completion for up to timeout"""
         # Properties are readonly
         pass
 
     async def get_datakey(self, source: str) -> DataKey:
         """Metadata like source, dtype, shape, precision, units"""
-        return self.describe_dict
+        return describedict_to_datakey(self.describe_dict)
 
-    async def get_reading(self) -> Reading:
+    async def get_reading(self) -> Reading[SignalDatatypeT]:
         dataset = CacheItem(
             value=self._prop_value, timestamp=self._secclient.conn_timestamp
         )
@@ -468,15 +457,15 @@ async def get_reading(self) -> Reading:
 
         return sec_reading.get_reading()
 
-    async def get_value(self) -> T:
+    async def get_value(self) -> SignalDatatypeT:
         dataset: Reading = await self.get_reading()
 
         return dataset["value"]  # type: ignore
 
-    async def get_setpoint(self) -> T:
+    async def get_setpoint(self) -> SignalDatatypeT:
         return await self.get_value()
 
-    def set_callback(self, callback: Callback[T] | None) -> None:
+    def set_callback(self, callback: Callback[Reading[SignalDatatypeT]] | None) -> None:
         pass
 
 
@@ -515,3 +504,27 @@ def secop_dtype_obj_from_json(prop_val):
         f"""unsupported datatype in Property:  {str(prop_val.__class__.__name__)}\n
         propval: {prop_val}"""
     )
+
+
+def describedict_to_datakey(describe_dict: dict) -> SECoPDataKey:
+    """Convert a DataKey to a SECoPDataKey"""
+    datakey = SECoPDataKey(
+        dtype=describe_dict["dtype"],
+        shape=describe_dict["shape"],
+        source=describe_dict["source"],
+        SECOP_datainfo=describe_dict["SECOP_datainfo"],
+    )
+
+    if "units" in describe_dict:
+        datakey["units"] = describe_dict["units"]
+
+    if "dtype_str" in describe_dict:
+        datakey["dtype_str"] = describe_dict["dtype_str"]
+
+    if "dtype_descr" in describe_dict:
+        datakey["dtype_descr"] = describe_dict["dtype_descr"]
+
+    if "dtype_numpy" in describe_dict:
+        datakey["dtype_numpy"] = describe_dict["dtype_numpy"]
+
+    return datakey

src/secop_ophyd/util.py

@@ -26,6 +26,7 @@
     TupleOf,
 )
 from ophyd_async.core._utils import StrictEnum
+from typing_extensions import NotRequired
 
 SCALAR_DATATYPES = (
     IntRange,
@@ -37,6 +38,32 @@
 )
 
 
+class SECoPDataKey(DataKey):
+    """
+    A DataKey that is used to describe the SECoP Datatype.
+    """
+
+    dtype_str: NotRequired[str]
+    """
+    The array-protocol typestring of the data-type object.
+    """
+
+    dtype_descr: NotRequired[list]
+    """
+    String representation of the numpy structured array dtype.
+    """
+
+    SECOP_datainfo: str
+    """
+    String representation of the original secop datatype.
+    """
+
+    SECoP_dtype: NotRequired[str]
+    """
+    The SECoP datatype of the data.
+    """
+
+
 class NestedRaggedArray(Exception):
     """The Datatype contains nested ragged arrays"""
 
@@ -552,7 +579,7 @@ def __init__(self, datatype: DataType) -> None:
         self.dtype_descr: list
 
         # string representation of the original secop datatype
-        self.secop_dtype_str = datatype.export_datatype()
+        self.secop_dtype_str = str(datatype.export_datatype())
 
         # Shape of Data
         self.shape = []
@@ -612,7 +639,8 @@ def __init__(self, datatype: DataType) -> None:
                 self.np_datatype = SECOP2DTYPE[datatype.__class__][0]
                 self.dtype = SECOP2DTYPE[datatype.__class__][1]
 
-    def get_datakey(self) -> DataKey:
+    def get_datakey(self) -> dict:
+
         describe_dict: dict = {}
         # Composite Datatypes & Arrays of COmposite Datatypes
         if self._is_composite:

tests/test_RE.py

@@ -17,7 +17,9 @@ async def test_run_engine_count(cryo_sim, run_engine, cryo_node: SECoPNodeDevice
 
     run = db[p]
 
-    data = run.primary.read()
+    print(run[0])
+
+    data = run[0].primary.read()
 
     cryo_dat = data[cryo_node.cryo.value.name]
 

tests/test_nested.py

@@ -61,7 +61,7 @@ async def test_nested_dtype_str_signal_generation(
 
     assert isinstance(val, np.ndarray)
     assert descr["dtype"] == "array"
-    assert descr["SECoP_dtype"] == "struct"
+    assert "struct" in descr["SECOP_datainfo"]
     await nested_node.disconnect_async()
 
 

tests/test_primitive_arrays.py

@@ -4,7 +4,6 @@
 from bluesky import RunEngine
 from bluesky.callbacks.best_effort import BestEffortCallback
 from bluesky.plans import count
-from bluesky.utils import ProgressBarManager
 from ophyd_async.core import SignalR
 
 from secop_ophyd.SECoPDevices import SECoPNodeDevice, SECoPReadableDevice
@@ -15,7 +14,6 @@ async def test_primitive_arrays(
 ):
     bec = BestEffortCallback()
     run_engine.subscribe(bec)
-    run_engine.waiting_hook = ProgressBarManager()
     run_engine.ignore_callback_exceptions = False
 
     prim_arr: SECoPReadableDevice = nested_node_re.primitive_arrays
@@ -57,7 +55,6 @@ async def test_primitive_float_array(
 ):
     bec = BestEffortCallback()
     run_engine.subscribe(bec)
-    run_engine.waiting_hook = ProgressBarManager()
     run_engine.ignore_callback_exceptions = False
 
     prim_arr: SECoPReadableDevice = nested_node_re.primitive_arrays