From 58cb87ad4f076a5edfecdf7cdb1c2ca1e067070e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 25 Nov 2025 20:19:58 +0100 Subject: [PATCH 01/26] Rewrite TEM server to allow second identical server that communicates as a camera --- instamaticServer/tem_server.py | 261 ++++++++++++++++++++------------- 1 file changed, 162 insertions(+), 99 deletions(-) diff --git a/instamaticServer/tem_server.py b/instamaticServer/tem_server.py index 13ee306..2796d22 100644 --- a/instamaticServer/tem_server.py +++ b/instamaticServer/tem_server.py @@ -1,86 +1,119 @@ import datetime +import logging import queue import socket import threading -import signal +import time import traceback -import logging from TEMController.microscope import get_microscope from serializer import dumper, loader from utils.config import config +from typing import Any, List, Tuple, Type -condition = threading.Condition() stop_program_event = threading.Event() -box = [] - _conf = config() -HOST = _conf.default_settings['tem_server_host'] -PORT = _conf.default_settings['tem_server_port'] BUFSIZE = 1024 +TIMEOUT = 0.5 +logfile = 'tem_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') +logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' +logging.basicConfig(level=logging.INFO, filename='tem_server.log', format=logging_fmt) -class TemServer(threading.Thread): - """TEM communcation server. - Takes a logger object `log`, command queue `q`, and name of the - microscope `name` that is used to initialize the connection to the - microscope. Start the server using `TemServer.run` which will wait - for items to appear on `q` and execute them on the specified - microscope instance. - """ - - def __init__(self, log=None, q=None, name=None): - super().__init__() +class DeviceServer(threading.Thread): + """General microscope / camera (Acquisition) communication server. - self._log = log - self._q = q + Takes a `name` of the microscope/camera and initializes appropriate device. + When `TemServer.start` thread method is called, `TemServer.run` starts. + The server will wait for cmd `requests` to appear in the queue, evaluate + them in order, and return the result via each client's `response_queue`. + """ - # self.name is a reserved parameter for threads - self._name = name + device_abbr: str + device_kind: str + requests: queue.Queue + responses: queue.Queue + host: str = 'localhost' + port: int + def __init__(self, name=None) -> None: + super().__init__() + self._name = name # self.name is a reserved parameter for threads + self.logger = logging.getLogger(self.device_abbr + 'S') # temS/camS server + self.tem = None self.verbose = False - def run(self): + def run(self) -> None: """Start the server thread.""" self.tem = get_microscope(name=self._name) self._name = self.tem.name - print("Initialized connection to microscope: %s" % (self._name)) + self.logger.info('Initialized %s %s server thread', self.device_kind, self._name) while True: now = datetime.datetime.now().strftime('%H:%M:%S.%f') - - cmd = self._q.get() - - with condition: - func_name = cmd['func_name'] - args = cmd.get('args', ()) - kwargs = cmd.get('kwargs', {}) - - try: - ret = self.evaluate(func_name, args, kwargs) - status = 200 - except Exception as e: - traceback.print_exc() - if self._log: - self._log.exception(e) - ret = (e.__class__.__name__, e.args) - status = 500 - - box.append((status, ret)) - condition.notify() - print("%s | %s %s: %s" % (now, status, func_name, ret)) - - def evaluate(self, func_name: str, args: list, kwargs: dict): + try: + cmd = self.requests.get(timeout=TIMEOUT) + except queue.Empty: + if stop_program_event.is_set(): + break + continue + + func_name = cmd.get('func_name', cmd.get('attr_name')) + args = cmd.get('args', ()) + kwargs = cmd.get('kwargs', {}) + + try: + ret = self.evaluate(func_name, args, kwargs) + status = 200 + except Exception as e: + traceback.print_exc() + self.logger.exception(e) + ret = (e.__class__.__name__, e.args) + status = 500 + + self.responses.put((status, ret)) + self.logger.info("%s | %s %s: %s", now, status, func_name, ret) + + self.logger.info('Terminating %s %s server thread', self.device_kind, self._name) + + def evaluate(self, func_name: str, args: list, kwargs: dict) -> Any: """Evaluate the function `func_name` on `self.tem` and call it with `args` and `kwargs`.""" - print(func_name, args, kwargs) + self.logger.debug('eval %s %s %s', func_name, args, kwargs) f = getattr(self.tem, func_name) ret = f(*args, **kwargs) return ret -def handle(conn, q): + +class TemServer(DeviceServer): + """TEM communcation server.""" + + device_abbr: str = 'tem' + device_kind: str = 'microscope' + requests = queue.Queue(maxsize=1) + responses = queue.Queue(maxsize=1) + host = _conf.default_settings['tem_server_host'] + port = _conf.default_settings['tem_server_port'] + + +class CamServer(DeviceServer): + """FEI Tecnai/Titan Acquisition camera communication server.""" + + device_abbr: str = 'cam' + device_kind: str = 'camera' + requests = queue.Queue(maxsize=1) + responses = queue.Queue(maxsize=1) + host = _conf.default_settings['cam_server_host'] + port = _conf.default_settings['cam_server_port'] + + def __init__(self, name=None) -> None: + super(CamServer, self).__init__(name=name) + self.logger.setLevel(logging.WARNING) + + +def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: """Handle incoming connection, put command on the Queue `q`, which is then handled by TEMServer.""" with conn: @@ -94,78 +127,108 @@ def handle(conn, q): data = loader(data) - if data == 'exit': + if data == 'exit' or data == 'kill': break - if data == 'kill': - break + server_type.requests.put(data) + response = server_type.responses.get() + conn.send(dumper(response)) - with condition: - q.put(data) - condition.wait() - response = box.pop() - conn.send(dumper(response)) +def listen(server_type: Type[DeviceServer]) -> None: + """Listen on a given server host/port and handle incoming instructions""" -def handle_kb_interrupt(sig, frame): - stop_program_event.set() + logger = logging.getLogger(server_type.device_abbr + 'L') # temL/camL listener + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as device_client: + device_client.bind((server_type.host, server_type.port)) + device_client.settimeout(TIMEOUT) + device_client.listen(0) + logger.info('Server listening on %s:%s', server_type.host, server_type.port) + while True: + if stop_program_event.is_set(): + break + try: + connection, _ = device_client.accept() + handle(connection, server_type) + except socket.timeout: + pass + except Exception as e: + logger.exception('Exception when handling connection: %s', e) + logger.info('Terminating %s listener thread', server_type.device_kind) -def main(): +def main() -> None: + """ + Connects to the TEM and starts a server for microscope communication. + Opens a socket on port {HOST}:{PORT}. - import argparse - description = """ -Connects to the TEM and starts a server for microscope communication. Opens a socket on port {HOST}:{PORT}. + This program initializes a connection to the TEM as defined in the config. + The purpose of this program is to isolate the microscope connection in + a separate process for improved stability of the interface in case + instamatic crashes or is started and stopped frequently. + For running the GUI, the temserver is required. + Another reason is that it allows for remote connections from different PCs. + The connection goes over a TCP socket. -This program initializes a connection to the TEM as defined in the config. The purpose of this program is to isolate the microscope connection in a separate process for improved stability of the interface in case instamatic crashes or is started and stopped frequently. For running the GUI, the temserver is required. Another reason is that it allows for remote connections from different PCs. The connection goes over a TCP socket. + The host and port are defined in `config/settings.yaml`. -The host and port are defined in `config/settings.yaml`. + The data sent over the socket is a serialized dictionary with the following: -The data sent over the socket is a serialized dictionary with the following elements: + - `func_name`: Name of the function to call (str) + - `args`: (Optional) List of arguments for the function (list) + - `kwargs`: (Optiona) Dictionary of keyword arguments for the function (dict) -- `func_name`: Name of the function to call (str) -- `args`: (Optional) List of arguments for the function (list) -- `kwargs`: (Optiona) Dictionary of keyword arguments for the function (dict) + The response is returned as a serialized object. + """ -The response is returned as a serialized object. -""" + import argparse - parser = argparse.ArgumentParser( - description=description, - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.add_argument('-t', '--microscope', action='store', dest='microscope', - help="""Override microscope to use.""") + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument('-t', '--microscope', action='store', + help='Override microscope to use.') + parser.add_argument('-c', '--camera', action='store_true', + help='If selected, start separate threads for a camera') parser.set_defaults(microscope=None) options = parser.parse_args() - microscope = options.microscope - logging.basicConfig(filename='tem_server.log', level=logging.INFO) + logging.info('Tecnai server starting') - q = queue.Queue(maxsize=100) + tem_server = TemServer(name=options.microscope) + tem_server.start() - tem_reader = TemServer(name=microscope, log=None, q=q) - tem_reader.start() - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind((HOST, PORT)) - s.listen(0) + tem_listener = threading.Thread(target=listen, args=(TemServer,)) + tem_listener.start() - logging.info("Server listening on %s:%s" % (HOST, PORT)) - print ("Server listening on %s:%s" % (HOST, PORT)) - - signal.signal(signal.SIGINT, handle_kb_interrupt) - - with s: - while True: - conn, addr = s.accept() - #logging.info('Connected by %s' % (addr)) -# print('Connected by', addr) - command_thread = threading.Thread(target=handle, args=(conn, q)) - command_thread.start() - command_thread.join() + threads = [tem_server, tem_listener] + if options.camera: + logging.info('Waiting for the TEM singleton to initialize') + for _ in range(100): + if getattr(tem_server, 'tem') is not None: # wait until TEM initialized + break + time.sleep(0.05) + else: # necessary check, Error extremely unlikely, TEM typically starts in ms + raise RuntimeError('Could not start TEM device on server in 5 seconds') + + cam_server = CamServer(name=None) + cam_server.start() + + cam_listener = threading.Thread(target=listen, args=(CamServer,)) + cam_listener.start() + + threads.extend([cam_server, cam_listener]) + + try: + while not stop_program_event.is_set(): time.sleep(TIMEOUT) + except KeyboardInterrupt: + logging.info("Received KeyboardInterrupt, shutting down...") + finally: + stop_program_event.set() + for thread in threads: + thread.join() + logging.info('Tecnai server terminating') + logging.shutdown() if __name__ == '__main__': From ea011464d6459d97d3eecad1af9e4c02ed7644fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Thu, 27 Nov 2025 19:00:55 +0100 Subject: [PATCH 02/26] Read camera config, add config interpreter, adapt get_camera, server --- instamaticServer/TEMController/microscope.py | 8 +- .../TEMController/tecnai_camera.py | 106 ++++++++++++++++++ instamaticServer/tem_server.py | 23 ++-- instamaticServer/utils/config.py | 42 +++++-- 4 files changed, 156 insertions(+), 23 deletions(-) create mode 100644 instamaticServer/TEMController/tecnai_camera.py diff --git a/instamaticServer/TEMController/microscope.py b/instamaticServer/TEMController/microscope.py index 7b0f0af..ae9ec50 100644 --- a/instamaticServer/TEMController/microscope.py +++ b/instamaticServer/TEMController/microscope.py @@ -3,7 +3,7 @@ _conf = config() _tem_interfaces = ('simulate', 'tecnai') -__all__ = ['get_microscope', 'get_microscope_class'] +__all__ = ['get_camera', 'get_microscope', 'get_microscope_class'] def get_microscope_class(interface: str): @@ -39,3 +39,9 @@ def get_microscope(name: str = None): tem = cls(name=name) return tem + + +def get_camera(name: str = None): + """Loads specifically the built-in camera interface, ignoring name.""" + from .tecnai_camera import TecnaiCamera + return TecnaiCamera(name=name) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py new file mode 100644 index 0000000..39d6528 --- /dev/null +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import atexit +import logging +from typing import Tuple, Any, Optional, List + +from tecnai_microscope import Singleton, TecnaiMicroscope +from utils.config import config + +try: + import numpy as np +except ImportError: + np = False + + +logger = logging.getLogger(__name__) + + +class TecnaiCamera(metaclass=Singleton): + """Interfaces any camera on an FEI Tecnai/Titan microscope.""" + + streamable = True + + # Set by `load_defaults` + camera_rotation_vs_stage_xy: float + default_binsize: int + default_exposure: float + dimensions: Tuple[int, int] + interface: str + possible_binsizes: List[int] + stretch_amplitude: float + stretch_azimuth: float + + def __init__(self, name='tecnai'): + """Initialize camera module.""" + self.name = name + self.load_defaults() + self.acq, self.cam = self.establish_connection() + logger.info(f'Camera Tecnai initialized') + atexit.register(self.release_connection) + + def __enter__(self): + self.establish_connection() + return self + + def __exit__(self, kind, value, traceback) -> None: + self.release_connection() + + def get_binning(self) -> int: + return self.default_binsize + + def get_camera_dimensions(self) -> Tuple[int, int]: + return self.dimensions + + def get_name(self) -> str: + return self.name + + def load_defaults(self) -> None: + _conf = config() + for key, val in _conf.camera.__dict__.items(): + setattr(self, key, val) + + def get_image(self, exposure: Optional[float] = None, binning: int = 1): + """Image acquisition interface.""" + self.cam.AcqParams.ExposureTime = exposure or self.default_exposure + self.cam.AcqParams.Binning = binning + img = self.acq.AcquireImages()[0] + sa = img.AsSafeArray + if np: + return np.stack(sa).T + return [[sa[r, c] for c in range(img.Height)] for r in range(img.Width)] + # try [[sa.GetElement([r, c]) or similar if direct indexing does not work... + + def get_image_dimensions(self) -> Tuple[int, int]: + """Get the binned dimensions reported by the camera.""" + return self.cam.ImageSize + + def get_movie( + self, + n_frames: int, + exposure: Optional[float] = None, + binsize: Optional[int] = None, + ): + """Unfortunately not designed to work with generators, as a server...""" + return [self.get_image(exposure, binsize) for _ in range(n_frames)] + + def establish_connection(self) -> Tuple[Any, Any]: + """Establish connection to the camera.""" + acq = TecnaiMicroscope()._tem.Acquisition() + acq.RemoveAllAcqDevices() + cam = acq.Cameras[0] + cam.AcqParams.ImageCorrection = 1 # bias and gain corr (0=off, 1=on) + cam.AcqParams.ImageSize = 0 # sub area centered (0=full, 1=half, 2=quarter) + acq.AddAcqDeviceByName(cam.Info.Name) + return acq, cam + + def release_connection(self) -> None: + """Release the connection to the camera.""" + pass + + +if __name__ == '__main__': + cam = TecnaiCamera() + from IPython import embed + + embed() diff --git a/instamaticServer/tem_server.py b/instamaticServer/tem_server.py index 2796d22..2994d92 100644 --- a/instamaticServer/tem_server.py +++ b/instamaticServer/tem_server.py @@ -6,10 +6,10 @@ import time import traceback -from TEMController.microscope import get_microscope +from TEMController.microscope import get_camera, get_microscope from serializer import dumper, loader from utils.config import config -from typing import Any, List, Tuple, Type +from typing import Any, Type, Callable stop_program_event = threading.Event() @@ -33,6 +33,7 @@ class DeviceServer(threading.Thread): device_abbr: str device_kind: str + device_getter: Callable requests: queue.Queue responses: queue.Queue host: str = 'localhost' @@ -42,13 +43,13 @@ def __init__(self, name=None) -> None: super().__init__() self._name = name # self.name is a reserved parameter for threads self.logger = logging.getLogger(self.device_abbr + 'S') # temS/camS server - self.tem = None + self.device = None self.verbose = False def run(self) -> None: """Start the server thread.""" - self.tem = get_microscope(name=self._name) - self._name = self.tem.name + self.device = self.device_getter(name=self._name) + self._name = self.device.name self.logger.info('Initialized %s %s server thread', self.device_kind, self._name) while True: @@ -79,10 +80,10 @@ def run(self) -> None: self.logger.info('Terminating %s %s server thread', self.device_kind, self._name) def evaluate(self, func_name: str, args: list, kwargs: dict) -> Any: - """Evaluate the function `func_name` on `self.tem` and call it with + """Evaluate the function `func_name` on `self.device` and call it with `args` and `kwargs`.""" self.logger.debug('eval %s %s %s', func_name, args, kwargs) - f = getattr(self.tem, func_name) + f = getattr(self.device, func_name) ret = f(*args, **kwargs) return ret @@ -92,6 +93,7 @@ class TemServer(DeviceServer): device_abbr: str = 'tem' device_kind: str = 'microscope' + device_getter = get_microscope requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) host = _conf.default_settings['tem_server_host'] @@ -103,6 +105,7 @@ class CamServer(DeviceServer): device_abbr: str = 'cam' device_kind: str = 'camera' + device_getter = get_camera requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) host = _conf.default_settings['cam_server_host'] @@ -120,13 +123,13 @@ def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: while True: if stop_program_event.is_set(): break - + data = conn.recv(BUFSIZE) if not data: break data = loader(data) - + if data == 'exit' or data == 'kill': break @@ -205,7 +208,7 @@ def main() -> None: if options.camera: logging.info('Waiting for the TEM singleton to initialize') for _ in range(100): - if getattr(tem_server, 'tem') is not None: # wait until TEM initialized + if getattr(tem_server, 'device') is not None: # wait until TEM initialized break time.sleep(0.05) else: # necessary check, Error extremely unlikely, TEM typically starts in ms diff --git a/instamaticServer/utils/config.py b/instamaticServer/utils/config.py index 52e7f84..27ac7c7 100644 --- a/instamaticServer/utils/config.py +++ b/instamaticServer/utils/config.py @@ -1,10 +1,25 @@ from pathlib import Path +from types import SimpleNamespace +from typing import Dict, Any + import yaml _settings_file = 'settings.yaml' +class NS(SimpleNamespace): + def get(self, key, default=None): + return getattr(self, key, default) + + +def dict_to_namespace(d: Dict) -> NS[Any]: + """Recursively converts a dictionary into a SimpleNamespace.""" + if isinstance(d, dict): + return NS(**{k: dict_to_namespace(v) for k, v in d.items()}) + return d + + class config: def __init__(self, name:str=None): @@ -14,25 +29,22 @@ def __init__(self, name:str=None): self.default_settings['microscope'] = name self.micr_interface, self.micr_wavelength, self.micr_ranges = self.microscope() + try: + self.camera = self.load_camera_config() + except FileNotFoundError: + self.camera = NS() def settings(self) -> dict: """load the settings.yaml file.""" - default = None - - direc = Path(__file__).resolve().parent - file = direc.joinpath(_settings_file) + directory = Path(__file__).resolve().parent + file = directory.joinpath(_settings_file) with open(str(file), 'r') as stream: - default = yaml.safe_load(stream) - - return default + return yaml.safe_load(stream) def microscope(self): """load the microscope.yaml file.""" - default = None - - direc = Path(__file__).resolve().parent - microscope_file = '\\' + str(self.default_settings['microscope']) + '.yaml' - file = str(direc) + microscope_file + directory = Path(__file__).resolve().parent + file = directory / (str(self.default_settings['microscope']) + '.yaml') with open(file, 'r') as stream: default = yaml.safe_load(stream) @@ -42,6 +54,12 @@ def microscope(self): return interface, wavelength, micr_ranges + def load_camera_config(self) -> NS: + directory = Path(__file__).resolve().parent + file = directory / (str(self.default_settings['camera']) + '.yaml') + with open(file, 'r') as stream: + return dict_to_namespace(yaml.safe_load(stream)) + if __name__ == '__main__': data = config() From 065cffb6371f32873372224b24c8909d9bc42e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Fri, 28 Nov 2025 15:34:14 +0100 Subject: [PATCH 03/26] Write 62 rudimentary tests for basic library functions, to be tested --- instamaticServer/utils/config.py | 2 +- tests.py | 410 +++++++++++++++++++++++++++++++ 2 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 tests.py diff --git a/instamaticServer/utils/config.py b/instamaticServer/utils/config.py index 27ac7c7..3dfee28 100644 --- a/instamaticServer/utils/config.py +++ b/instamaticServer/utils/config.py @@ -13,7 +13,7 @@ def get(self, key, default=None): return getattr(self, key, default) -def dict_to_namespace(d: Dict) -> NS[Any]: +def dict_to_namespace(d: Dict) -> NS: """Recursively converts a dictionary into a SimpleNamespace.""" if isinstance(d, dict): return NS(**{k: dict_to_namespace(v) for k, v in d.items()}) diff --git a/tests.py b/tests.py new file mode 100644 index 0000000..1225d60 --- /dev/null +++ b/tests.py @@ -0,0 +1,410 @@ +import atexit +import socket +import threading +import time +import unittest +from typing import Any, Dict + +from instamaticServer.utils.config import NS, config, dict_to_namespace + +_conf_dict = {'a': 1, 'b': {'c': 3, 'd': 4}} +_conf = config() +PRECISION_NM = 250 +PRECISION_DEG = 0.1 + + +class TestConfig(unittest.TestCase): + def test_namespace(self): + ns = NS(_conf_dict) + self.assertEqual(ns.a, 1) + self.assertEqual(ns.b, {'c': 3, 'd': 4}) + + def test_dict_to_namespace(self): + ns = dict_to_namespace(_conf_dict) + self.assertEqual(ns.a, 1) + self.assertEqual(ns.b, NS({'c': 3, 'd': 4})) + + def test_config(self): + global _conf + self.assertIn(_conf.micr_interface, {'tecnai', 'simulate'}) + self.assertIsInstance(_conf.camera, NS) + + +class TestSerializers(unittest.TestCase): + + def test_json_serializer(self): + from instamaticServer.serializer import json_dumper, json_loader + self.assertEqual(_conf, json_dumper(json_loader(_conf))) + + def test_pickle_serializer(self): + from instamaticServer.serializer import pickle_dumper, pickle_loader + self.assertEqual(_conf, pickle_dumper(pickle_loader(_conf))) + + def test_msgpack_serializer(self): + from instamaticServer.serializer import msgpack_dumper, msgpack_loader + self.assertEqual(_conf, msgpack_dumper(msgpack_loader(_conf))) + + +class TestServer(unittest.TestCase): + def test_00_init_tem_server(self): + from instamaticServer.tem_server import TemServer, listen + self.tem_server = TemServer() + self.tem_server.start() + self.tem_listener = threading.Thread(target=listen, args=(TemServer,)) + self.tem_listener.start() + self.threads = [self.tem_server, self.tem_listener] + for _ in range(100): + if getattr(self.tem_server, 'device') is not None: + break + time.sleep(0.05) + else: + raise RuntimeError('Could not start TEM device on server in 5 seconds') + self.const = self.tem_server.device._tem_constant + + def test_01_client_tem_connect(self): + host = _conf.default_settings['tem_server_host'] + port = _conf.default_settings['tem_server_port'] + self.socket_tem = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket_tem.connect((host, port)) + atexit.register(self.socket_tem.close) + + @staticmethod + def socket_send(socket, d): + from instamaticServer.serializer import dumper, loader + from instamaticServer.utils.exceptions import TEMCommunicationError, exception_list + if d.get('args', None) is None: + d['args'] = () + if d.get('kwargs', None) is None: + d['kwargs'] = {} + socket.send(dumper(d)) + response = socket.recv(1024) + if response: + status, data = loader(response) + else: + raise RuntimeError(f'Received empty response when evaluating {d}') + if status == 200: + return data + elif status == 500: + error_code, args = data + raise exception_list.get(error_code, TEMCommunicationError)(*args) + else: + raise ConnectionError(f'Unknown status code: {status}') + + def tem_send(self, d: Dict[str, Any]): + return self.socket_send(self.socket_tem, d) + + def test_20_getHolderType(self): + r = self.tem_send({'func_name': 'getHolderType'}) + self.assertIsInstance(r, self.const.StageHolderType) + + def test_21_getStagePosition(self): + r = self.tem_send({'func_name': 'getStagePosition'}) + self.assertIsInstance(r, tuple) + self.assertEqual(len(r), 5) + + def test_22_getStageSpeed(self): + r = self.tem_send({'func_name': 'getStageSpeed'}) + self.assertEqual(r, 0.5) + + def test_23_is_goniotool_available(self): + r = self.tem_send({'func_name': 'is_goniotool_available'}) + self.assertEqual(r, False) + + def test_24_isAThreadAlive(self): + r = self.tem_send({'func_name': 'isAThreadAlive'}) + self.assertEqual(r, False) + + def test_25_isStageMoving(self): + r = self.tem_send({'func_name': 'isStageMoving'}) + self.assertEqual(r, False) + + def test_30_setStagePosition(self): + p = (0, 0, 0, 0, 0) + self.tem_send({'func_name': 'setStagePosition', 'args': p}) + r = self.tem_send({'func_name': 'getStagePosition'}) + self.assertAlmostEqual(r[0], p[0], delta=PRECISION_NM) + self.assertAlmostEqual(r[1], p[1], delta=PRECISION_NM) + self.assertAlmostEqual(r[2], p[2], delta=PRECISION_NM) + self.assertAlmostEqual(r[3], p[3], delta=PRECISION_DEG) + self.assertAlmostEqual(r[4], p[4], delta=PRECISION_DEG) + + def test_31_setStagePosition(self): + p = {'x': 10_000, 'y': 10_000} + self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) + r = self.tem_send({'func_name': 'getStagePosition'}) + self.assertAlmostEqual(r[0], 10_000, delta=PRECISION_NM) + self.assertAlmostEqual(r[1], 10_000, delta=PRECISION_NM) + self.assertAlmostEqual(r[2], 0, delta=PRECISION_NM) + self.assertAlmostEqual(r[3], 0, delta=PRECISION_DEG) + self.assertAlmostEqual(r[4], 0, delta=PRECISION_DEG) + + def test_32_setStagePosition(self): + p = {'z': 10_000} + self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) + r = self.tem_send({'func_name': 'getStagePosition'}) + self.assertAlmostEqual(r[0], 10_000, delta=PRECISION_NM) + self.assertAlmostEqual(r[1], 10_000, delta=PRECISION_NM) + self.assertAlmostEqual(r[2], 10_000, delta=PRECISION_NM) + + def test_33_setStagePosition(self): + self.tem_send({'func_name': 'setStagePosition', 'args': (0, 0, 0, 0, 0)}) + p = {'a': 10} + self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) + r = self.tem_send({'func_name': 'getStagePosition'}) + self.assertAlmostEqual(r[3], 10, delta=PRECISION_DEG) + + def test_35_setStagePosition(self): + d = {'func_name': 'setStagePosition'} + self.tem_send({**d, 'args': (0, 0, 0, 0, 0)}) + t0 = time.perf_counter() + self.tem_send({**d, 'kwargs': {'x': 10_000}}) + t1 = time.perf_counter() + self.tem_send({**d, 'kwargs': {'x': 0, 'speed': 0.1}}) + t2 = time.perf_counter() + self.assertLess(t1 - t0, t2 - t1) + self.tem_send({**d, 'kwargs': {'x': 10_000, 'speed': 0.05}}) + t3 = time.perf_counter() + self.assertLess(t2 - t1, t3 - t2) + self.tem_send({**d, 'kwargs': {'x': 0, 'speed': 0.02}}) + t4 = time.perf_counter() + self.assertLess(t3 - t2, t4 - t3) + + def test_36_setStagePosition(self): + d = {'func_name': 'setStagePosition'} + self.tem_send({**d, 'args': (0, 0, 0, 0, 0)}) + t0 = time.perf_counter() + self.tem_send({**d, 'kwargs': {'a': 5}}) + t1 = time.perf_counter() + self.tem_send({**d, 'kwargs': {'a': 0, 'speed': 0.1}}) + t2 = time.perf_counter() + self.assertLess(t1 - t0, t2 - t1) + self.tem_send({**d, 'kwargs': {'a': 5, 'speed': 0.05}}) + t3 = time.perf_counter() + self.assertLess(t2 - t1, t3 - t2) + self.tem_send({**d, 'kwargs': {'a': 0, 'speed': 0.02}}) + t4 = time.perf_counter() + self.assertLess(t3 - t2, t4 - t3) + + def test_38_setStagePosition(self): + self.tem_send({'func_name': 'setStagePosition', 'args': (0, 0, 0, 0, 0)}) + p = {'a': 30, 'wait': False} + t0 = time.perf_counter() + self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) + t1 = time.perf_counter() + self.tem_send({'func_name': 'waitForStage', 'kwargs': {'delay': 0.01}}) + t2 = time.perf_counter() + q = {'a': 0, 'wait': True} + self.tem_send({'func_name': 'setStagePosition', 'kwargs': q}) + t3 = time.perf_counter() + self.assertLess(t1 - t0, t2 - t1) + self.assertLess(t1 - t0, t3 - t2) + + def test_40_setStageA(self): + p = (0, ) + self.tem_send({'func_name': 'setStageA', 'args': p}) + r = self.tem_send({'func_name': 'getStagePosition'}) + self.assertAlmostEqual(r[3], p[0], delta=PRECISION_DEG) + + def test_41_setStageA(self): + p = (10, ) + self.tem_send({'func_name': 'setStageA', 'args': p}) + r = self.tem_send({'func_name': 'getStagePosition'}) + self.assertAlmostEqual(r[3], 0, delta=PRECISION_DEG) + + def test_46_setRotationSpeed(self): + d = {'func_name': 'setStageA'} + self.tem_send({**d, 'kwargs': {'a': 0}}) + t0 = time.perf_counter() + self.tem_send({'func_name': 'setRotationSpeed', 'args': (1.0, )}) + self.tem_send({**d, 'kwargs': {'a': 5}}) + t1 = time.perf_counter() + self.tem_send({'func_name': 'setRotationSpeed', 'args': (0.1, )}) + self.tem_send({**d, 'kwargs': {'a': 0}}) + t2 = time.perf_counter() + self.assertLess(t1 - t0, t2 - t1) + self.tem_send({'func_name': 'setRotationSpeed', 'args': (0.05, )}) + self.tem_send({**d, 'kwargs': {'a': 5}}) + t3 = time.perf_counter() + self.assertLess(t2 - t1, t3 - t2) + self.tem_send({'func_name': 'setRotationSpeed', 'args': (0.02, )}) + self.tem_send({**d, 'kwargs': {'a': 0}}) + t4 = time.perf_counter() + self.assertLess(t3 - t2, t4 - t3) + s = self.tem_send({'func_name': 'setRotationSpeed'}) + self.assertEqual(s, 0.02) + self.tem_send({'func_name': 'setRotationSpeed', 'args': (1.0, )}) + + def test_48_setStageA(self): + self.tem_send({'func_name': 'setStageA', 'args': (0, )}) + p = {'a': 10, 'wait': False} + t0 = time.perf_counter() + self.tem_send({'func_name': 'setStageA', 'kwargs': p}) + t1 = time.perf_counter() + self.tem_send({'func_name': 'waitForStage', 'kwargs': {'delay': 0.01}}) + t2 = time.perf_counter() + q = {'a': 0, 'wait': True} + self.tem_send({'func_name': 'setStageA', 'kwargs': q}) + t3 = time.perf_counter() + self.assertLess(t1 - t0, t2 - t1) + self.assertLess(t1 - t0, t3 - t2) + + def test_50_getGunShift(self): + self.tem_send({'func_name': 'getGunShift'}) + + def test_51_getHTValue(self): + self.tem_send({'func_name': 'getHTValue'}) + + def test_52_isBeamBlanked(self): + self.tem_send({'func_name': 'isBeamBlanked'}) + + def test_53_getBeamAlignShift(self): + self.tem_send({'func_name': 'getBeamAlignShift'}) + + def test_54_getSpotSize(self): + self.tem_send({'func_name': 'getSpotSize'}) + + def test_55_getBrightness(self): + self.tem_send({'func_name': 'getBrightness'}) + + def test_56_getBrightnessValue(self): + self.tem_send({'func_name': 'getBrightnessValue'}) + + def test_57_getBeamShift(self): + self.tem_send({'func_name': 'getBeamShift'}) + + def test_58_getBeamTilt(self): + self.tem_send({'func_name': 'getBeamTilt'}) + + def test_59_getCondensorLensStigmator(self): + self.tem_send({'func_name': 'getCondensorLensStigmator'}) + + def test_61_getScreenCurrent(self): + self.tem_send({'func_name': 'getScreenCurrent'}) + + def test_62_isfocusscreenin(self): + r = self.tem_send({'func_name': 'isfocusscreenin'}) + self.assertIsInstance(r, bool) + + def test_63_getScreenPosition(self): + r = self.tem_send({'func_name': 'getScreenPosition'}) + self.assertIn(r, {'up', 'down', ''}) + + def test_64_getDiffFocus(self): + r = self.tem_send({'func_name': 'getDiffFocus'}) + self.assertGreater(r, 0) + self.assertLess(r, 65536) + + def test_65_getDiffFocusValue(self): + r = self.tem_send({'func_name': 'getDiffFocusValue'}) + self.assertGreater(r, -1.0) + self.assertLess(r, 1.0) + + def test_65_getFocus(self): + r = self.tem_send({'func_name': 'getFocus'}) + self.assertGreater(r, -1.0) + self.assertLess(r, 1.0) + + def test_67_FunctionMode(self): + r = self.tem_send({'func_name': 'getFunctionMode'}) + r = self.tem_send({'func_name': 'setFunctionMode', 'args': (r, )}) + self.assertIn(r, ('lowmag', 'mag1', 'samag', 'mag2', 'diff')) + + def test_68_Magnification(self): + r = self.tem_send({'func_name': 'getMagnification'}) + r = self.tem_send({'func_name': 'setMagnification', 'args': (r, )}) + self.assertIsInstance(r, float) + + def test_69_MagnificationIndex(self): + r = self.tem_send({'func_name': 'getMagnificationIndex'}) + r = self.tem_send({'func_name': 'setMagnificationIndex', 'args': (r, )}) + self.assertIsInstance(r, int) + + def test_70_getDarkFieldTilt(self): + r = self.tem_send({'func_name': 'getDarkFieldTilt'}) + self.assertIsInstance(r[0], float) + + def test_71_getImageShift1(self): + r = self.tem_send({'func_name': 'getImageShift1'}) + self.assertIsInstance(r[0], float) + + def test_72_getImageShift2(self): + r = self.tem_send({'func_name': 'getImageShift1'}) + self.assertEqual(r[0], 0) + + def test_73_getImageBeamShift(self): + r = self.tem_send({'func_name': 'getImageBeamShift'}) + self.assertIsInstance(r[0], float) + + def test_74_getDiffShift(self): + r = self.tem_send({'func_name': 'getDiffShift'}) + self.assertIsInstance(r[0], float) + + def test_76_getObjectiveLensStigmator(self): + r = self.tem_send({'func_name': 'getObjectiveLensStigmator'}) + self.assertIsInstance(r[0], float) + + def test_77_getIntermediateLensStigmator(self): + r = self.tem_send({'func_name': 'getIntermediateLensStigmator'}) + self.assertIsInstance(r[0], float) + + def test_80_init_cam_server(self): + from instamaticServer.tem_server import CamServer, listen + self.cam_server = CamServer() + self.cam_server.start() + self.cam_listener = threading.Thread(target=listen, args=(CamServer,)) + self.cam_listener.start() + self.threads.extend([self.cam_server, self.cam_listener]) + + def test_81_client_cam_connect(self): + host = _conf.default_settings['cam_server_host'] + port = _conf.default_settings['cam_server_port'] + self.socket_cam = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket_cam.connect((host, port)) + atexit.register(self.socket_cam.close) + + def cam_send(self, d: Dict[str, Any]): + return self.socket_send(self.socket_cam, d) + + def test_90_get_binning(self): + r = self.cam_send({'attr_name': 'get_binning'}) + self.assertIsInstance(r, int) + + def test_91_default_binsize(self): + r = self.cam_send({'attr_name': 'get_binning'}) + s = self.cam_send({'attr_name': 'default_binsize'}) + self.assertEqual(r, s) + + def test_92_dimensions(self): + r = self.cam_send({'attr_name': 'dimensions'}) + self.assertIsInstance(r[0], int) + + def test_93_get_camera_dimensions(self): + r = self.cam_send({'attr_name': 'get_camera_dimensions'}) + s = self.cam_send({'attr_name': 'dimensions'}) + self.assertEqual(r, s) + + def test_94_get_image_dimensions(self): + r = self.cam_send({'attr_name': 'get_image_dimensions'}) + s = self.cam_send({'attr_name': 'get_camera_dimensions'}) + self.assertEqual(r, s) + + def test_96_get_image(self): + r = self.cam_send({'attr_name': 'get_image'}) + r = self.cam_send({'attr_name': 'get_image_dimensions'}) + self.assertEqual(len(r)) + + def test_97_get_image(self): + r = self.cam_send({'attr_name': 'get_image'}) + s = self.cam_send({'attr_name': 'get_image_dimensions'}) + self.assertEqual(len(r), s[0]) + + def test_98_get_movie(self): + r = self.cam_send({'attr_name': 'get_movie'}) + s = self.cam_send({'attr_name': 'get_image_dimensions'}) + self.assertEqual(len(r[0]), s[0]) + + def test_99_shutdown(self): + from instamaticServer.tem_server import stop_program_event + stop_program_event.set() + for thread in [self.tem_server, self.tem_listener]: + thread.join() From 816e19395d4858955d6a680feecf593a0241e3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 2 Dec 2025 18:41:18 +0100 Subject: [PATCH 04/26] Reimplement the whole freaking system as a package to enable testing --- instamaticServer/TEMController/camera.py | 38 ++ instamaticServer/TEMController/microscope.py | 28 +- instamaticServer/TEMController/simu_camera.py | 108 +++++ .../TEMController/simu_microscope.py | 8 +- .../TEMController/tecnai_camera.py | 21 +- .../TEMController/tecnai_microscope.py | 20 +- .../TEMController/tecnai_stage_thread.py | 16 +- instamaticServer/serializer.py | 2 +- instamaticServer/tem_server.py | 82 ++-- instamaticServer/utils/config.py | 8 +- instamaticServer/utils/settings.yaml | 8 +- instamaticServer/utils/simulate_cam.yaml | 21 + .../{simulate.yaml => simulate_tem.yaml} | 20 +- instamaticServer/utils/singleton.py | 7 + .../typing.py => utils/types.py} | 0 tests.py | 377 +++++++++--------- 16 files changed, 484 insertions(+), 280 deletions(-) create mode 100644 instamaticServer/TEMController/camera.py create mode 100644 instamaticServer/TEMController/simu_camera.py create mode 100644 instamaticServer/utils/simulate_cam.yaml rename instamaticServer/utils/{simulate.yaml => simulate_tem.yaml} (98%) create mode 100644 instamaticServer/utils/singleton.py rename instamaticServer/{TEMController/typing.py => utils/types.py} (100%) diff --git a/instamaticServer/TEMController/camera.py b/instamaticServer/TEMController/camera.py new file mode 100644 index 0000000..95c4761 --- /dev/null +++ b/instamaticServer/TEMController/camera.py @@ -0,0 +1,38 @@ +from instamaticServer.utils.config import config + + +_conf = config() +_cam_interfaces = ('simulate', 'tecnai') + +__all__ = ['get_camera', 'get_camera_class'] + + +def get_camera_class(interface: str): + """Grab the cam class with the given 'interface'.""" + + if interface == 'simulate': + from .simu_camera import SimuCamera as CamCls + elif interface == 'tecnai': + from .tecnai_microscope import TecnaiMicroscope as CamCls + else: + raise ValueError("No such microscope interface: %s" % interface) + + return CamCls + + +def get_camera(name: str = None): + """Return an instance of camera interface `tecnai` or `simulate`""" + + if name in _cam_interfaces: + interface = name + else: + interface = _conf.camera.interface + name = _conf.default_settings['microscope'] + + cls = get_camera_class(interface=interface) + tem = cls(name=name) + + return tem + + + diff --git a/instamaticServer/TEMController/microscope.py b/instamaticServer/TEMController/microscope.py index ae9ec50..c1c6fb3 100644 --- a/instamaticServer/TEMController/microscope.py +++ b/instamaticServer/TEMController/microscope.py @@ -1,34 +1,28 @@ -from utils.config import config +from instamaticServer.utils.config import config + _conf = config() _tem_interfaces = ('simulate', 'tecnai') -__all__ = ['get_camera', 'get_microscope', 'get_microscope_class'] +__all__ = ['get_microscope', 'get_microscope_class'] def get_microscope_class(interface: str): """Grab the tem class with the given 'interface'.""" if interface == 'simulate': - from .simu_microscope import SimuMicroscope as cls + from .simu_microscope import SimuMicroscope as TemCls elif interface == 'tecnai': - from .tecnai_microscope import TecnaiMicroscope as cls + from .tecnai_microscope import TecnaiMicroscope as TemCls else: - raise ValueError("No such microscope interface: %s" % (interface)) + raise ValueError("No such microscope interface: %s" % interface) - return cls + return TemCls def get_microscope(name: str = None): - """Generic class to load microscope interface class. - - name: str - Specify which microscope to use, must be one of `tecnai`, `simulate` - use_server: bool - Connect to microscope server running on the host/port defined in the config file + """Return an instance of microscope interface `tecnai` or `simulate`""" - returns: TEM interface class - """ if name in _tem_interfaces: interface = name else: @@ -39,9 +33,3 @@ def get_microscope(name: str = None): tem = cls(name=name) return tem - - -def get_camera(name: str = None): - """Loads specifically the built-in camera interface, ignoring name.""" - from .tecnai_camera import TecnaiCamera - return TecnaiCamera(name=name) diff --git a/instamaticServer/TEMController/simu_camera.py b/instamaticServer/TEMController/simu_camera.py new file mode 100644 index 0000000..440eb14 --- /dev/null +++ b/instamaticServer/TEMController/simu_camera.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import atexit +import logging +import time +from typing import Tuple, Any, Optional, List + +from instamaticServer.utils.config import config +from instamaticServer.utils.singleton import Singleton + +try: + import numpy as np +except ImportError: + np = False + + +logger = logging.getLogger(__name__) + + +class SimuCamera(metaclass=Singleton): + """Simple class that simulates the camera interface and mocks the method calls.""" + + streamable = True + + # Set by `load_defaults` + camera_rotation_vs_stage_xy = None # type: float + default_binsize = None # type: int + default_exposure = None # type: float + dimensions = None # type: Tuple[int, int] + interface = None # type: str + possible_binsizes = None # type: List[int] + stretch_amplitude = None # type: float + stretch_azimuth = None # type: float + + def __init__(self, name='simulate'): + """Initialize camera module.""" + self.name = name + self.load_defaults() + self.establish_connection() + logger.info(f'Camera simulate initialized') + atexit.register(self.release_connection) + + def __enter__(self): + self.establish_connection() + return self + + def __exit__(self, kind, value, traceback) -> None: + self.release_connection() + + def get_binning(self) -> int: + return self.default_binsize + + def get_camera_dimensions(self) -> Tuple[int, int]: + return self.dimensions + + def get_name(self) -> str: + return self.name + + def load_defaults(self) -> None: + _conf = config() + for key, val in _conf.camera.__dict__.items(): + setattr(self, key, val) + + def get_image(self, exposure: Optional[float] = None, binsize: int = 1): + """Image acquisition interface.""" + exposure = exposure or self.default_exposure + binsize = binsize or self.default_binsize + t0 = time.perf_counter() + dx, dy = self.dimensions + dx, dy = dx // binsize, dy // binsize + if np: + img = 256 * np.random.random_sample((dx, dy)) + else: + import random + v = list(range(0, 256)) + img = [random.sample(v, dx) for _ in range(dy)] + while time.perf_counter() - t0 < exposure: + time.sleep(0.001) + return img + + def get_image_dimensions(self) -> Tuple[int, int]: + """Get the binned dimensions reported by the camera.""" + dx, dy = self.dimensions + return dx // self.default_binsize, dy // self.default_binsize + + def get_movie( + self, + n_frames: int, + exposure: Optional[float] = None, + binsize: Optional[int] = None, + ): + """Unfortunately not designed to work with generators, as a server...""" + return [self.get_image(exposure, binsize) for _ in range(n_frames)] + + def establish_connection(self) -> Tuple[Any, Any]: + """Establish connection to the camera.""" + pass + + def release_connection(self) -> None: + """Release the connection to the camera.""" + pass + + +if __name__ == '__main__': + cam = SimuCamera() + from IPython import embed + + embed() diff --git a/instamaticServer/TEMController/simu_microscope.py b/instamaticServer/TEMController/simu_microscope.py index 62bce77..d840c3c 100644 --- a/instamaticServer/TEMController/simu_microscope.py +++ b/instamaticServer/TEMController/simu_microscope.py @@ -2,9 +2,9 @@ import time from typing import Optional, Tuple, Union -from .typing import StagePositionTuple, float_deg, int_nm -from utils.exceptions import TEMValueError -from utils.config import config +from instamaticServer.utils.config import config +from instamaticServer.utils.exceptions import TEMValueError +from instamaticServer.utils.types import StagePositionTuple, float_deg, int_nm NTRLMAPPING = { @@ -535,7 +535,7 @@ def setIntermediateLensStigmator(self, x: int, y: int): self.intermediatelensstigmator_y = y def getObjectiveLensStigmator(self) -> Tuple[int, int]: - return self.objectivelensstigmator_x, self.objectivelensstigmatir_y + return self.objectivelensstigmator_x, self.objectivelensstigmator_y def setObjectiveLensStigmator(self, x: int, y: int): self.objectivelensstigmator_x = x diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index 39d6528..e19eddf 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -4,8 +4,9 @@ import logging from typing import Tuple, Any, Optional, List -from tecnai_microscope import Singleton, TecnaiMicroscope -from utils.config import config +from instamaticServer.TEMController.tecnai_microscope import TecnaiMicroscope +from instamaticServer.utils.config import config +from instamaticServer.utils.singleton import Singleton try: import numpy as np @@ -22,14 +23,14 @@ class TecnaiCamera(metaclass=Singleton): streamable = True # Set by `load_defaults` - camera_rotation_vs_stage_xy: float - default_binsize: int - default_exposure: float - dimensions: Tuple[int, int] - interface: str - possible_binsizes: List[int] - stretch_amplitude: float - stretch_azimuth: float + camera_rotation_vs_stage_xy = None # type: float + default_binsize = None # type: int + default_exposure = None # type: float + dimensions = None # type: Tuple[int, int] + interface = None # type: str + possible_binsizes = None # type: List[int] + stretch_amplitude = None # type: float + stretch_azimuth = None # type: float def __init__(self, name='tecnai'): """Initialize camera module.""" diff --git a/instamaticServer/TEMController/tecnai_microscope.py b/instamaticServer/TEMController/tecnai_microscope.py index 457b01e..7d9767f 100644 --- a/instamaticServer/TEMController/tecnai_microscope.py +++ b/instamaticServer/TEMController/tecnai_microscope.py @@ -5,10 +5,11 @@ from math import pi from typing import Optional -from .typing import StagePositionTuple, float_deg, int_nm -from utils.exceptions import FEIValueError, TEMCommunicationError -from utils.config import config -from TEMController.tecnai_stage_thread import TecnaiStageThread +from instamaticServer.TEMController.tecnai_stage_thread import TecnaiStageThread +from instamaticServer.utils.config import config +from instamaticServer.utils.exceptions import FEIValueError, TEMCommunicationError +from instamaticServer.utils.singleton import Singleton +from instamaticServer.utils.types import StagePositionTuple, float_deg, int_nm _FUNCTION_MODES = {1: 'lowmag', 2: 'mag1', 3: 'samag', 4: 'mag2', 5: 'LAD', 6: 'diff'} @@ -25,15 +26,6 @@ # ('Mh', [440000, 520000, 610000, 700000, 780000, 910000])]) -class Singleton(type): - """Singleton Metaclass from Stack Overflow, stackoverflow.com/q/6760685""" - _instances = {} - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] - - class TecnaiMicroscope(metaclass=Singleton): """Python bindings to the Tecnai-G2 microscope using the COM scripting interface.""" @@ -59,7 +51,7 @@ def __init__(self, name: str=None) -> None: time.sleep(1) self._t += 1 if self._t > 3: - print('Waiting for microscope, t = %ss' % (self._t)) + print('Waiting for microscope, t = %ss' % self._t) if self._t > 30: raise TEMCommunicationError('Cannot establish microscope connection (timeout).') diff --git a/instamaticServer/TEMController/tecnai_stage_thread.py b/instamaticServer/TEMController/tecnai_stage_thread.py index 518b733..51672dd 100644 --- a/instamaticServer/TEMController/tecnai_stage_thread.py +++ b/instamaticServer/TEMController/tecnai_stage_thread.py @@ -2,13 +2,23 @@ import comtypes from typing import Union + +from instamaticServer.utils.types import StagePositionTuple + + class TecnaiStageThread(threading.Thread): """ Stage communication with the Tecnai microscope over a separate thread. """ - def __init__(self, tem=None, pos:(float, float, float, float, float)=None, axis:int=None, speed:Union[int, float]=0): - super().__init__() + def __init__( + self, + tem = None, + pos: StagePositionTuple = None, + axis: int = None, + speed: Union[int, float] = 0, + ): + super().__init__(name='TecnaiStageThread') #TEM-Scriptinginterface self._tem = tem @@ -59,7 +69,7 @@ def run(self) -> None: self._tem.Stage.GoToWithSpeed(stagePos, self._axis, self._speed) class ContextManagedComtypes(): - '''The Context Manager Protocoll is used to initialize the COM connection again''' + """The Context Manager Protocol is used to initialize the COM connection again""" def __enter__(self): comtypes.CoInitialize() return self diff --git a/instamaticServer/serializer.py b/instamaticServer/serializer.py index ede6283..3b408b9 100644 --- a/instamaticServer/serializer.py +++ b/instamaticServer/serializer.py @@ -1,7 +1,7 @@ import json import pickle -from utils.config import config +from instamaticServer.utils.config import config _conf = config() PROTOCOL = _conf.default_settings['tem_communication_protocol'] diff --git a/instamaticServer/tem_server.py b/instamaticServer/tem_server.py index 2994d92..70aa666 100644 --- a/instamaticServer/tem_server.py +++ b/instamaticServer/tem_server.py @@ -6,11 +6,14 @@ import time import traceback -from TEMController.microscope import get_camera, get_microscope -from serializer import dumper, loader -from utils.config import config from typing import Any, Type, Callable +from instamaticServer.TEMController.camera import get_camera +from instamaticServer.TEMController.microscope import get_microscope +from instamaticServer.serializer import dumper, loader +from instamaticServer.utils.config import config + + stop_program_event = threading.Event() _conf = config() @@ -31,26 +34,26 @@ class DeviceServer(threading.Thread): them in order, and return the result via each client's `response_queue`. """ - device_abbr: str - device_kind: str - device_getter: Callable - requests: queue.Queue - responses: queue.Queue - host: str = 'localhost' - port: int - - def __init__(self, name=None) -> None: - super().__init__() - self._name = name # self.name is a reserved parameter for threads + device_abbr = None # type: str + device_kind = None # type: str + device_getter = None # type: Callable + requests = None # type: queue.Queue + responses = None # type: queue.Queue + host = 'localhost' # type: str + port = None # type: int + + def __init__(self, name = None) -> None: + super().__init__(name=self.device_kind + '_server') + self.interface_name = name self.logger = logging.getLogger(self.device_abbr + 'S') # temS/camS server self.device = None self.verbose = False def run(self) -> None: """Start the server thread.""" - self.device = self.device_getter(name=self._name) - self._name = self.device.name - self.logger.info('Initialized %s %s server thread', self.device_kind, self._name) + self.device = self.device_getter(name=self.interface_name) + self.device.get_attrs = self.get_attrs + self.logger.info('Initialized %s %s server thread', self.device_kind, self.device.name) while True: now = datetime.datetime.now().strftime('%H:%M:%S.%f') @@ -84,16 +87,27 @@ def evaluate(self, func_name: str, args: list, kwargs: dict) -> Any: `args` and `kwargs`.""" self.logger.debug('eval %s %s %s', func_name, args, kwargs) f = getattr(self.device, func_name) - ret = f(*args, **kwargs) - return ret + return f(*args, **kwargs) if callable(f) else f + + def get_attrs(self): + """Get attributes from cam object to update __dict__ on client side.""" + attrs = {} + for item in dir(self.device): + if item.startswith('_'): + continue + obj = getattr(self.device, item) + if not callable(obj): + attrs[item] = type(obj) + + return attrs class TemServer(DeviceServer): """TEM communcation server.""" - device_abbr: str = 'tem' - device_kind: str = 'microscope' - device_getter = get_microscope + device_abbr = 'tem' + device_kind = 'microscope' + device_getter = staticmethod(get_microscope) requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) host = _conf.default_settings['tem_server_host'] @@ -103,9 +117,9 @@ class TemServer(DeviceServer): class CamServer(DeviceServer): """FEI Tecnai/Titan Acquisition camera communication server.""" - device_abbr: str = 'cam' - device_kind: str = 'camera' - device_getter = get_camera + device_abbr = 'cam' + device_kind = 'camera' + device_getter = staticmethod(get_camera) requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) host = _conf.default_settings['cam_server_host'] @@ -120,11 +134,16 @@ def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: """Handle incoming connection, put command on the Queue `q`, which is then handled by TEMServer.""" with conn: + conn.settimeout(TIMEOUT) while True: if stop_program_event.is_set(): break - data = conn.recv(BUFSIZE) + try: + data = conn.recv(BUFSIZE) + except socket.timeout: + continue + if not data: break @@ -135,7 +154,8 @@ def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: server_type.requests.put(data) response = server_type.responses.get() - conn.send(dumper(response)) + serialized = dumper(response) + conn.sendall(serialized) def listen(server_type: Type[DeviceServer]) -> None: @@ -145,7 +165,7 @@ def listen(server_type: Type[DeviceServer]) -> None: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as device_client: device_client.bind((server_type.host, server_type.port)) device_client.settimeout(TIMEOUT) - device_client.listen(0) + device_client.listen(1) logger.info('Server listening on %s:%s', server_type.host, server_type.port) while True: if stop_program_event.is_set(): @@ -200,7 +220,7 @@ def main() -> None: tem_server = TemServer(name=options.microscope) tem_server.start() - tem_listener = threading.Thread(target=listen, args=(TemServer,)) + tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') tem_listener.start() threads = [tem_server, tem_listener] @@ -214,10 +234,10 @@ def main() -> None: else: # necessary check, Error extremely unlikely, TEM typically starts in ms raise RuntimeError('Could not start TEM device on server in 5 seconds') - cam_server = CamServer(name=None) + cam_server = CamServer(name=options.microscope) cam_server.start() - cam_listener = threading.Thread(target=listen, args=(CamServer,)) + cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') cam_listener.start() threads.extend([cam_server, cam_listener]) diff --git a/instamaticServer/utils/config.py b/instamaticServer/utils/config.py index 3dfee28..29b98e4 100644 --- a/instamaticServer/utils/config.py +++ b/instamaticServer/utils/config.py @@ -1,6 +1,6 @@ from pathlib import Path from types import SimpleNamespace -from typing import Dict, Any +from typing import Dict import yaml @@ -25,7 +25,7 @@ class config: def __init__(self, name:str=None): self.default_settings = self.settings() - if name != None: + if name is not None: self.default_settings['microscope'] = name self.micr_interface, self.micr_wavelength, self.micr_ranges = self.microscope() @@ -45,7 +45,7 @@ def microscope(self): """load the microscope.yaml file.""" directory = Path(__file__).resolve().parent file = directory / (str(self.default_settings['microscope']) + '.yaml') - with open(file, 'r') as stream: + with open(str(file), 'r') as stream: default = yaml.safe_load(stream) interface = default['interface'] @@ -57,7 +57,7 @@ def microscope(self): def load_camera_config(self) -> NS: directory = Path(__file__).resolve().parent file = directory / (str(self.default_settings['camera']) + '.yaml') - with open(file, 'r') as stream: + with open(str(file), 'r') as stream: return dict_to_namespace(yaml.safe_load(stream)) diff --git a/instamaticServer/utils/settings.yaml b/instamaticServer/utils/settings.yaml index 8e0ff49..fd8e986 100644 --- a/instamaticServer/utils/settings.yaml +++ b/instamaticServer/utils/settings.yaml @@ -1,5 +1,5 @@ -microscope: tecnaiG2 -camera: simulate +microscope: simulate_tem +camera: simulate_cam calibration: simulate # Global toggle to force simulated camera/microscope interface @@ -11,10 +11,10 @@ flatfield: # Run the TEM connection in a different process (recommended) use_tem_server: True -tem_server_host: '169.254.178.125' +tem_server_host: 'localhost' tem_server_port: 8088 tem_require_admin: False -tem_communication_protocol: 'pickle' # pickle, json, msgpack, yaml +tem_communication_protocol: pickle # pickle, json, msgpack, yaml # Run the Camera connection in a different process use_cam_server: False diff --git a/instamaticServer/utils/simulate_cam.yaml b/instamaticServer/utils/simulate_cam.yaml new file mode 100644 index 0000000..957c591 --- /dev/null +++ b/instamaticServer/utils/simulate_cam.yaml @@ -0,0 +1,21 @@ +calib_beamshift: + gridsize: 5 + stepsize: 500 +calib_directbeam: + BeamShift: + gridsize: 5 + stepsize: 75 + DiffShift: + gridsize: 5 + stepsize: 300 +camera_rotation_vs_stage_xy: -2.24 +default_binsize: 1 +default_exposure: 0.1 +dimensions: [512, 512] +dynamic_range: 255 +interface: simulate +physical_pixelsize: 0.055 +possible_binsizes: [1, 2, 4, 8, 16, 32, 64, 128, 256] +stretch_amplitude: 0.0 +stretch_azimuth: 0.0 +dead_time: 0.0 \ No newline at end of file diff --git a/instamaticServer/utils/simulate.yaml b/instamaticServer/utils/simulate_tem.yaml similarity index 98% rename from instamaticServer/utils/simulate.yaml rename to instamaticServer/utils/simulate_tem.yaml index eb8eb59..7944877 100644 --- a/instamaticServer/utils/simulate.yaml +++ b/instamaticServer/utils/simulate_tem.yaml @@ -1,10 +1,10 @@ -interface: simulate -ranges: - diff: [150, 200, 250, 300, 400, 500, 600, 800, 1000, 1200, 1500, 2000, 2500, 3000, - 3500, 4000, 4500] - lowmag: [50, 80, 100, 150, 200, 250, 300, 400, 500, 600, 800, 1000, 1200, 1500, - 2000, 2500, 3000, 5000, 6000, 8000, 10000, 12000, 15000] - mag1: [2500, 3000, 4000, 5000, 6000, 8000, 10000, 12000, 15000, 20000, 25000, 30000, - 40000, 50000, 60000, 80000, 100000, 120000, 150000, 200000, 250000, 300000, 400000, - 500000, 600000, 800000, 1000000, 1500000, 2000000] -wavelength: 0.025079 +interface: simulate +ranges: + diff: [150, 200, 250, 300, 400, 500, 600, 800, 1000, 1200, 1500, 2000, 2500, 3000, + 3500, 4000, 4500] + lowmag: [50, 80, 100, 150, 200, 250, 300, 400, 500, 600, 800, 1000, 1200, 1500, + 2000, 2500, 3000, 5000, 6000, 8000, 10000, 12000, 15000] + mag1: [2500, 3000, 4000, 5000, 6000, 8000, 10000, 12000, 15000, 20000, 25000, 30000, + 40000, 50000, 60000, 80000, 100000, 120000, 150000, 200000, 250000, 300000, 400000, + 500000, 600000, 800000, 1000000, 1500000, 2000000] +wavelength: 0.025079 diff --git a/instamaticServer/utils/singleton.py b/instamaticServer/utils/singleton.py new file mode 100644 index 0000000..42b593c --- /dev/null +++ b/instamaticServer/utils/singleton.py @@ -0,0 +1,7 @@ +class Singleton(type): + """Singleton Metaclass from Stack Overflow, stackoverflow.com/q/6760685""" + _instances = {} + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] diff --git a/instamaticServer/TEMController/typing.py b/instamaticServer/utils/types.py similarity index 100% rename from instamaticServer/TEMController/typing.py rename to instamaticServer/utils/types.py diff --git a/tests.py b/tests.py index 1225d60..61c4a46 100644 --- a/tests.py +++ b/tests.py @@ -1,4 +1,6 @@ import atexit +import json +import pickle import socket import threading import time @@ -6,23 +8,25 @@ from typing import Any, Dict from instamaticServer.utils.config import NS, config, dict_to_namespace +from instamaticServer.tem_server import stop_program_event _conf_dict = {'a': 1, 'b': {'c': 3, 'd': 4}} _conf = config() PRECISION_NM = 250 PRECISION_DEG = 0.1 +TIMEOUT = 5 class TestConfig(unittest.TestCase): def test_namespace(self): - ns = NS(_conf_dict) + ns = NS(**_conf_dict) self.assertEqual(ns.a, 1) self.assertEqual(ns.b, {'c': 3, 'd': 4}) def test_dict_to_namespace(self): ns = dict_to_namespace(_conf_dict) self.assertEqual(ns.a, 1) - self.assertEqual(ns.b, NS({'c': 3, 'd': 4})) + self.assertEqual(ns.b, NS(**{'c': 3, 'd': 4})) def test_config(self): global _conf @@ -31,97 +35,137 @@ def test_config(self): class TestSerializers(unittest.TestCase): - def test_json_serializer(self): from instamaticServer.serializer import json_dumper, json_loader - self.assertEqual(_conf, json_dumper(json_loader(_conf))) + self.assertEqual(_conf_dict, json_loader(json_dumper(_conf_dict))) def test_pickle_serializer(self): from instamaticServer.serializer import pickle_dumper, pickle_loader - self.assertEqual(_conf, pickle_dumper(pickle_loader(_conf))) + self.assertEqual(_conf_dict, pickle_loader(pickle_dumper(_conf_dict))) def test_msgpack_serializer(self): - from instamaticServer.serializer import msgpack_dumper, msgpack_loader - self.assertEqual(_conf, msgpack_dumper(msgpack_loader(_conf))) + try: + from instamaticServer.serializer import msgpack_dumper, msgpack_loader + self.assertEqual(_conf_dict, msgpack_loader(msgpack_dumper(_conf_dict))) + except ImportError: + pass class TestServer(unittest.TestCase): - def test_00_init_tem_server(self): - from instamaticServer.tem_server import TemServer, listen - self.tem_server = TemServer() - self.tem_server.start() - self.tem_listener = threading.Thread(target=listen, args=(TemServer,)) - self.tem_listener.start() - self.threads = [self.tem_server, self.tem_listener] - for _ in range(100): - if getattr(self.tem_server, 'device') is not None: - break + + @classmethod + def setUpClass(cls) -> None: + from instamaticServer.tem_server import CamServer, TemServer, listen + cls.tem_server = TemServer() + cls.tem_server.start() + cls.tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') + cls.tem_listener.start() + cls.threads = [cls.tem_server, cls.tem_listener] + + t0 = time.perf_counter() + while getattr(cls.tem_server, 'device', None) is None: + if time.perf_counter() - t0 > 5: + raise RuntimeError("Server device did not initialize") time.sleep(0.05) - else: - raise RuntimeError('Could not start TEM device on server in 5 seconds') - self.const = self.tem_server.device._tem_constant - def test_01_client_tem_connect(self): - host = _conf.default_settings['tem_server_host'] - port = _conf.default_settings['tem_server_port'] - self.socket_tem = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.socket_tem.connect((host, port)) - atexit.register(self.socket_tem.close) + cls.cam_server = CamServer() + cls.cam_server.start() + cls.cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') + cls.cam_listener.start() + cls.threads.extend([cls.cam_server, cls.cam_listener]) + + try: + cls.const = cls.tem_server.device._tem_constant + except AttributeError: + cls.const = None + + tem_host = _conf.default_settings['tem_server_host'] + tem_port = _conf.default_settings['tem_server_port'] + cls.socket_tem = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + cls.socket_tem.connect((tem_host, tem_port)) + cls.socket_tem.settimeout(TIMEOUT) + # atexit.register(cls.socket_tem.close) + + cam_host = _conf.default_settings['cam_server_host'] + cam_port = _conf.default_settings['cam_server_port'] + cls.socket_cam = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + cls.socket_cam.connect((cam_host, cam_port)) + cls.socket_cam.settimeout(TIMEOUT) + # atexit.register(cls.socket_cam.close) + + @classmethod + def tearDownClass(cls) -> None: + stop_program_event.set() + for thread in cls.threads: + thread.join(timeout=5) + if thread.is_alive(): + print(f"Thread {thread.name} did not exit") + cls.socket_tem.close() @staticmethod - def socket_send(socket, d): + def socket_send(s: socket.socket, func: str, args = (), kwargs = None): from instamaticServer.serializer import dumper, loader from instamaticServer.utils.exceptions import TEMCommunicationError, exception_list - if d.get('args', None) is None: - d['args'] = () - if d.get('kwargs', None) is None: - d['kwargs'] = {} - socket.send(dumper(d)) - response = socket.recv(1024) + kwargs = kwargs or {} + d = {'func_name': func, 'args': args, 'kwargs': kwargs} + buffer_size = 1024 + if func in ('get_image', 'get_movie'): + buffer_size += 8 * _conf.camera.dimensions[0] * _conf.camera.dimensions[1] + s.send(dumper(d)) + response = s.recv(buffer_size) if response: - status, data = loader(response) + for _ in range(10): # warrants all image/movie is collected + try: + status, data = loader(response) + except (pickle.UnpicklingError, json.JSONDecodeError, RuntimeError): + response += s.recv(buffer_size) + else: + break else: - raise RuntimeError(f'Received empty response when evaluating {d}') + raise RuntimeError('Received empty response when evaluating %s' % d) if status == 200: return data elif status == 500: error_code, args = data raise exception_list.get(error_code, TEMCommunicationError)(*args) else: - raise ConnectionError(f'Unknown status code: {status}') + raise ConnectionError('Unknown status code: %s' % status) - def tem_send(self, d: Dict[str, Any]): - return self.socket_send(self.socket_tem, d) + def tem_send(self, func: str, args = (), kwargs = None): + return self.socket_send(self.socket_tem, func, args, kwargs) + + def cam_send(self, func: str, args = (), kwargs = None): + return self.socket_send(self.socket_cam, func, args, kwargs) def test_20_getHolderType(self): - r = self.tem_send({'func_name': 'getHolderType'}) + r = self.tem_send('getHolderType') self.assertIsInstance(r, self.const.StageHolderType) def test_21_getStagePosition(self): - r = self.tem_send({'func_name': 'getStagePosition'}) + r = self.tem_send('getStagePosition') self.assertIsInstance(r, tuple) self.assertEqual(len(r), 5) def test_22_getStageSpeed(self): - r = self.tem_send({'func_name': 'getStageSpeed'}) + r = self.tem_send('getStageSpeed') self.assertEqual(r, 0.5) def test_23_is_goniotool_available(self): - r = self.tem_send({'func_name': 'is_goniotool_available'}) + r = self.tem_send('is_goniotool_available') self.assertEqual(r, False) def test_24_isAThreadAlive(self): - r = self.tem_send({'func_name': 'isAThreadAlive'}) + r = self.tem_send('isAThreadAlive') self.assertEqual(r, False) def test_25_isStageMoving(self): - r = self.tem_send({'func_name': 'isStageMoving'}) + r = self.tem_send('isStageMoving') self.assertEqual(r, False) def test_30_setStagePosition(self): p = (0, 0, 0, 0, 0) - self.tem_send({'func_name': 'setStagePosition', 'args': p}) - r = self.tem_send({'func_name': 'getStagePosition'}) + self.tem_send('setStagePosition', p) + r = self.tem_send('getStagePosition') self.assertAlmostEqual(r[0], p[0], delta=PRECISION_NM) self.assertAlmostEqual(r[1], p[1], delta=PRECISION_NM) self.assertAlmostEqual(r[2], p[2], delta=PRECISION_NM) @@ -129,282 +173,257 @@ def test_30_setStagePosition(self): self.assertAlmostEqual(r[4], p[4], delta=PRECISION_DEG) def test_31_setStagePosition(self): - p = {'x': 10_000, 'y': 10_000} - self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) - r = self.tem_send({'func_name': 'getStagePosition'}) - self.assertAlmostEqual(r[0], 10_000, delta=PRECISION_NM) - self.assertAlmostEqual(r[1], 10_000, delta=PRECISION_NM) + p = {'x': 10000, 'y': 10000} + self.tem_send('setStagePosition', kwargs=p) + r = self.tem_send('getStagePosition') + self.assertAlmostEqual(r[0], 10000, delta=PRECISION_NM) + self.assertAlmostEqual(r[1], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[2], 0, delta=PRECISION_NM) self.assertAlmostEqual(r[3], 0, delta=PRECISION_DEG) self.assertAlmostEqual(r[4], 0, delta=PRECISION_DEG) def test_32_setStagePosition(self): - p = {'z': 10_000} - self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) - r = self.tem_send({'func_name': 'getStagePosition'}) - self.assertAlmostEqual(r[0], 10_000, delta=PRECISION_NM) - self.assertAlmostEqual(r[1], 10_000, delta=PRECISION_NM) - self.assertAlmostEqual(r[2], 10_000, delta=PRECISION_NM) + p = {'z': 10000} + self.tem_send('setStagePosition', kwargs=p) + r = self.tem_send('getStagePosition') + self.assertAlmostEqual(r[0], 10000, delta=PRECISION_NM) + self.assertAlmostEqual(r[1], 10000, delta=PRECISION_NM) + self.assertAlmostEqual(r[2], 10000, delta=PRECISION_NM) def test_33_setStagePosition(self): - self.tem_send({'func_name': 'setStagePosition', 'args': (0, 0, 0, 0, 0)}) + self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) p = {'a': 10} - self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) - r = self.tem_send({'func_name': 'getStagePosition'}) + self.tem_send('setStagePosition', kwargs=p) + r = self.tem_send('getStagePosition') self.assertAlmostEqual(r[3], 10, delta=PRECISION_DEG) def test_35_setStagePosition(self): - d = {'func_name': 'setStagePosition'} - self.tem_send({**d, 'args': (0, 0, 0, 0, 0)}) + self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send({**d, 'kwargs': {'x': 10_000}}) + self.tem_send('setStagePosition', kwargs={'x': 10000}) t1 = time.perf_counter() - self.tem_send({**d, 'kwargs': {'x': 0, 'speed': 0.1}}) + self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.1}) t2 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) - self.tem_send({**d, 'kwargs': {'x': 10_000, 'speed': 0.05}}) + self.tem_send('setStagePosition', kwargs={'x': 10000, 'speed': 0.05}) t3 = time.perf_counter() self.assertLess(t2 - t1, t3 - t2) - self.tem_send({**d, 'kwargs': {'x': 0, 'speed': 0.02}}) + self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.02}) t4 = time.perf_counter() self.assertLess(t3 - t2, t4 - t3) def test_36_setStagePosition(self): - d = {'func_name': 'setStagePosition'} - self.tem_send({**d, 'args': (0, 0, 0, 0, 0)}) + self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send({**d, 'kwargs': {'a': 5}}) + self.tem_send('setStagePosition', kwargs={'a': 5}) t1 = time.perf_counter() - self.tem_send({**d, 'kwargs': {'a': 0, 'speed': 0.1}}) + self.tem_send('setStagePosition', kwargs={'a': 0, 'speed': 0.1}) t2 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) - self.tem_send({**d, 'kwargs': {'a': 5, 'speed': 0.05}}) + self.tem_send('setStagePosition', kwargs={'a': 5, 'speed': 0.05}) t3 = time.perf_counter() self.assertLess(t2 - t1, t3 - t2) - self.tem_send({**d, 'kwargs': {'a': 0, 'speed': 0.02}}) + self.tem_send('setStagePosition', kwargs={'a': 0, 'speed': 0.02}) t4 = time.perf_counter() self.assertLess(t3 - t2, t4 - t3) def test_38_setStagePosition(self): - self.tem_send({'func_name': 'setStagePosition', 'args': (0, 0, 0, 0, 0)}) - p = {'a': 30, 'wait': False} + self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send({'func_name': 'setStagePosition', 'kwargs': p}) + self.tem_send('setStagePosition', kwargs={'a': 30, 'wait': False}) t1 = time.perf_counter() - self.tem_send({'func_name': 'waitForStage', 'kwargs': {'delay': 0.01}}) + self.tem_send('waitForStage', kwargs={'delay': 0.01}) t2 = time.perf_counter() q = {'a': 0, 'wait': True} - self.tem_send({'func_name': 'setStagePosition', 'kwargs': q}) + self.tem_send('setStagePosition', kwargs=q) t3 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) self.assertLess(t1 - t0, t3 - t2) def test_40_setStageA(self): p = (0, ) - self.tem_send({'func_name': 'setStageA', 'args': p}) - r = self.tem_send({'func_name': 'getStagePosition'}) + self.tem_send('setStageA', p) + r = self.tem_send('getStagePosition') self.assertAlmostEqual(r[3], p[0], delta=PRECISION_DEG) - def test_41_setStageA(self): - p = (10, ) - self.tem_send({'func_name': 'setStageA', 'args': p}) - r = self.tem_send({'func_name': 'getStagePosition'}) - self.assertAlmostEqual(r[3], 0, delta=PRECISION_DEG) - def test_46_setRotationSpeed(self): - d = {'func_name': 'setStageA'} - self.tem_send({**d, 'kwargs': {'a': 0}}) + self.tem_send('setStageA', kwargs={'value': 0}) t0 = time.perf_counter() - self.tem_send({'func_name': 'setRotationSpeed', 'args': (1.0, )}) - self.tem_send({**d, 'kwargs': {'a': 5}}) + self.tem_send('setRotationSpeed', (1.0, )) + self.tem_send('setStageA', kwargs={'value': 5}) t1 = time.perf_counter() - self.tem_send({'func_name': 'setRotationSpeed', 'args': (0.1, )}) - self.tem_send({**d, 'kwargs': {'a': 0}}) + self.tem_send('setRotationSpeed', (0.1, )) + self.tem_send('setStageA', kwargs={'value': 0}) t2 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) - self.tem_send({'func_name': 'setRotationSpeed', 'args': (0.05, )}) - self.tem_send({**d, 'kwargs': {'a': 5}}) + self.tem_send('setRotationSpeed', (0.05, )) + self.tem_send('setStageA', kwargs={'value': 5}) t3 = time.perf_counter() self.assertLess(t2 - t1, t3 - t2) - self.tem_send({'func_name': 'setRotationSpeed', 'args': (0.02, )}) - self.tem_send({**d, 'kwargs': {'a': 0}}) + self.tem_send('setRotationSpeed', (0.02, )) + self.tem_send('setStageA', kwargs={'value': 0}) t4 = time.perf_counter() self.assertLess(t3 - t2, t4 - t3) - s = self.tem_send({'func_name': 'setRotationSpeed'}) + s = self.tem_send('setRotationSpeed') self.assertEqual(s, 0.02) - self.tem_send({'func_name': 'setRotationSpeed', 'args': (1.0, )}) + self.tem_send('setRotationSpeed', (1.0, )) def test_48_setStageA(self): - self.tem_send({'func_name': 'setStageA', 'args': (0, )}) - p = {'a': 10, 'wait': False} + self.tem_send('setStageA', (0, )) t0 = time.perf_counter() - self.tem_send({'func_name': 'setStageA', 'kwargs': p}) + self.tem_send('setStageA', kwargs={'value': 10, 'wait': False}) t1 = time.perf_counter() - self.tem_send({'func_name': 'waitForStage', 'kwargs': {'delay': 0.01}}) + self.tem_send('waitForStage', kwargs={'delay': 0.01}) t2 = time.perf_counter() - q = {'a': 0, 'wait': True} - self.tem_send({'func_name': 'setStageA', 'kwargs': q}) + self.tem_send('setStageA', kwargs={'value': 0, 'wait': True}) t3 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) self.assertLess(t1 - t0, t3 - t2) def test_50_getGunShift(self): - self.tem_send({'func_name': 'getGunShift'}) + self.tem_send('getGunShift') def test_51_getHTValue(self): - self.tem_send({'func_name': 'getHTValue'}) + self.tem_send('getHTValue') def test_52_isBeamBlanked(self): - self.tem_send({'func_name': 'isBeamBlanked'}) + self.tem_send('isBeamBlanked') def test_53_getBeamAlignShift(self): - self.tem_send({'func_name': 'getBeamAlignShift'}) + self.tem_send('getBeamAlignShift') def test_54_getSpotSize(self): - self.tem_send({'func_name': 'getSpotSize'}) + self.tem_send('getSpotSize') def test_55_getBrightness(self): - self.tem_send({'func_name': 'getBrightness'}) + self.tem_send('getBrightness') def test_56_getBrightnessValue(self): - self.tem_send({'func_name': 'getBrightnessValue'}) + self.tem_send('getBrightnessValue') def test_57_getBeamShift(self): - self.tem_send({'func_name': 'getBeamShift'}) + self.tem_send('getBeamShift') def test_58_getBeamTilt(self): - self.tem_send({'func_name': 'getBeamTilt'}) + self.tem_send('getBeamTilt') def test_59_getCondensorLensStigmator(self): - self.tem_send({'func_name': 'getCondensorLensStigmator'}) + self.tem_send('getCondensorLensStigmator') def test_61_getScreenCurrent(self): - self.tem_send({'func_name': 'getScreenCurrent'}) + self.tem_send('getScreenCurrent') def test_62_isfocusscreenin(self): - r = self.tem_send({'func_name': 'isfocusscreenin'}) + r = self.tem_send('isfocusscreenin') self.assertIsInstance(r, bool) def test_63_getScreenPosition(self): - r = self.tem_send({'func_name': 'getScreenPosition'}) + r = self.tem_send('getScreenPosition') self.assertIn(r, {'up', 'down', ''}) def test_64_getDiffFocus(self): - r = self.tem_send({'func_name': 'getDiffFocus'}) + r = self.tem_send('getDiffFocus') self.assertGreater(r, 0) self.assertLess(r, 65536) def test_65_getDiffFocusValue(self): - r = self.tem_send({'func_name': 'getDiffFocusValue'}) + func_mode = self.tem_send('getFunctionMode') + self.tem_send('setFunctionMode', 'diff') + r = self.tem_send('getDiffFocusValue') + print(func_mode) + self.tem_send('setFunctionMode', (func_mode, )) self.assertGreater(r, -1.0) self.assertLess(r, 1.0) def test_65_getFocus(self): - r = self.tem_send({'func_name': 'getFocus'}) + r = self.tem_send('getFocus') self.assertGreater(r, -1.0) self.assertLess(r, 1.0) def test_67_FunctionMode(self): - r = self.tem_send({'func_name': 'getFunctionMode'}) - r = self.tem_send({'func_name': 'setFunctionMode', 'args': (r, )}) + r = self.tem_send('getFunctionMode') + self.tem_send('setFunctionMode', (r, )) self.assertIn(r, ('lowmag', 'mag1', 'samag', 'mag2', 'diff')) def test_68_Magnification(self): - r = self.tem_send({'func_name': 'getMagnification'}) - r = self.tem_send({'func_name': 'setMagnification', 'args': (r, )}) - self.assertIsInstance(r, float) + r = self.tem_send('getMagnification') + self.tem_send('setMagnification', (r, )) + self.assertIsInstance(r, (int, float)) def test_69_MagnificationIndex(self): - r = self.tem_send({'func_name': 'getMagnificationIndex'}) - r = self.tem_send({'func_name': 'setMagnificationIndex', 'args': (r, )}) + r = self.tem_send('getMagnificationIndex') + self.tem_send('setMagnificationIndex', (r, )) self.assertIsInstance(r, int) def test_70_getDarkFieldTilt(self): - r = self.tem_send({'func_name': 'getDarkFieldTilt'}) + r = self.tem_send('getDarkFieldTilt') self.assertIsInstance(r[0], float) def test_71_getImageShift1(self): - r = self.tem_send({'func_name': 'getImageShift1'}) - self.assertIsInstance(r[0], float) + r = self.tem_send('getImageShift1') + self.assertIsInstance(r[0], (int, float)) + self.assertIsInstance(r[1], (int, float)) def test_72_getImageShift2(self): - r = self.tem_send({'func_name': 'getImageShift1'}) - self.assertEqual(r[0], 0) + r = self.tem_send('getImageShift2') + self.assertIsInstance(r[0], (int, float)) + self.assertIsInstance(r[1], (int, float)) def test_73_getImageBeamShift(self): - r = self.tem_send({'func_name': 'getImageBeamShift'}) - self.assertIsInstance(r[0], float) + r = self.tem_send('getImageBeamShift') + self.assertIsInstance(r[0], (int, float)) + self.assertIsInstance(r[1], (int, float)) def test_74_getDiffShift(self): - r = self.tem_send({'func_name': 'getDiffShift'}) - self.assertIsInstance(r[0], float) + r = self.tem_send('getDiffShift') + self.assertIsInstance(r[0], (int, float)) + self.assertIsInstance(r[1], (int, float)) def test_76_getObjectiveLensStigmator(self): - r = self.tem_send({'func_name': 'getObjectiveLensStigmator'}) - self.assertIsInstance(r[0], float) + r = self.tem_send('getObjectiveLensStigmator') + self.assertIsInstance(r[0], (int, float)) + self.assertIsInstance(r[1], (int, float)) def test_77_getIntermediateLensStigmator(self): - r = self.tem_send({'func_name': 'getIntermediateLensStigmator'}) - self.assertIsInstance(r[0], float) - - def test_80_init_cam_server(self): - from instamaticServer.tem_server import CamServer, listen - self.cam_server = CamServer() - self.cam_server.start() - self.cam_listener = threading.Thread(target=listen, args=(CamServer,)) - self.cam_listener.start() - self.threads.extend([self.cam_server, self.cam_listener]) - - def test_81_client_cam_connect(self): - host = _conf.default_settings['cam_server_host'] - port = _conf.default_settings['cam_server_port'] - self.socket_cam = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.socket_cam.connect((host, port)) - atexit.register(self.socket_cam.close) - - def cam_send(self, d: Dict[str, Any]): - return self.socket_send(self.socket_cam, d) + r = self.tem_send('getIntermediateLensStigmator') + self.assertIsInstance(r[0], (int, float)) + self.assertIsInstance(r[1], (int, float)) def test_90_get_binning(self): - r = self.cam_send({'attr_name': 'get_binning'}) + r = self.cam_send('get_binning') self.assertIsInstance(r, int) def test_91_default_binsize(self): - r = self.cam_send({'attr_name': 'get_binning'}) - s = self.cam_send({'attr_name': 'default_binsize'}) + r = self.cam_send('get_binning') + s = self.cam_send('default_binsize') self.assertEqual(r, s) def test_92_dimensions(self): - r = self.cam_send({'attr_name': 'dimensions'}) + r = self.cam_send('dimensions') self.assertIsInstance(r[0], int) def test_93_get_camera_dimensions(self): - r = self.cam_send({'attr_name': 'get_camera_dimensions'}) - s = self.cam_send({'attr_name': 'dimensions'}) + r = self.cam_send('get_camera_dimensions') + s = self.cam_send('dimensions') self.assertEqual(r, s) def test_94_get_image_dimensions(self): - r = self.cam_send({'attr_name': 'get_image_dimensions'}) - s = self.cam_send({'attr_name': 'get_camera_dimensions'}) - self.assertEqual(r, s) + r = self.cam_send('get_image_dimensions') + s = self.cam_send('get_camera_dimensions') + self.assertEqual(r[0], s[0]) + self.assertEqual(r[1], s[1]) def test_96_get_image(self): - r = self.cam_send({'attr_name': 'get_image'}) - r = self.cam_send({'attr_name': 'get_image_dimensions'}) - self.assertEqual(len(r)) + r = self.cam_send('get_image') + s = self.cam_send('get_image_dimensions') + self.assertEqual(len(r), s[0]) def test_97_get_image(self): - r = self.cam_send({'attr_name': 'get_image'}) - s = self.cam_send({'attr_name': 'get_image_dimensions'}) + r = self.cam_send('get_image') + s = self.cam_send('get_image_dimensions') self.assertEqual(len(r), s[0]) def test_98_get_movie(self): - r = self.cam_send({'attr_name': 'get_movie'}) - s = self.cam_send({'attr_name': 'get_image_dimensions'}) + r = self.cam_send('get_movie', (1, )) + s = self.cam_send('get_image_dimensions') self.assertEqual(len(r[0]), s[0]) - - def test_99_shutdown(self): - from instamaticServer.tem_server import stop_program_event - stop_program_event.set() - for thread in [self.tem_server, self.tem_listener]: - thread.join() From 607f37103d529835cb15b949cc2b76fda9f5661c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 2 Dec 2025 19:38:06 +0100 Subject: [PATCH 05/26] Adapt simu microscope and fix all 50 tests --- .../TEMController/simu_microscope.py | 83 ++++++++++++++++++- .../TEMController/tecnai_microscope.py | 1 - tests.py | 65 ++++++++------- 3 files changed, 115 insertions(+), 34 deletions(-) diff --git a/instamaticServer/TEMController/simu_microscope.py b/instamaticServer/TEMController/simu_microscope.py index d840c3c..0db98f8 100644 --- a/instamaticServer/TEMController/simu_microscope.py +++ b/instamaticServer/TEMController/simu_microscope.py @@ -58,12 +58,18 @@ def __init__(self, name: str = None): self.BeamTilt_x = random.randint(MIN, MAX) self.BeamTilt_y = random.randint(MIN, MAX) + self.IlluminationTiltX = random.randint(MIN, MAX) + self.IlluminationTiltY = random.randint(MIN, MAX) + self.ImageShift1_x = random.randint(MIN, MAX) self.ImageShift1_y = random.randint(MIN, MAX) self.ImageShift2_x = random.randint(MIN, MAX) self.ImageShift2_y = random.randint(MIN, MAX) + self.ImageBeamShift_x = random.randint(MIN, MAX) + self.ImageBeamShift_y = random.randint(MIN, MAX) + # self.FunctionMode_value = random.randint(0, 2) self.FunctionMode_value = 0 @@ -140,7 +146,7 @@ def __init__(self, name: str = None): 'current': current, 'is_moving': False, 'speed': speed, - 'speed_setting': 12, + 'speed_setting': 1, 'direction': +1, 'start': 0.0, 'end': 0.0, @@ -255,12 +261,25 @@ def getCurrentDensity(self) -> float: rand_val = (random.random() - 0.5) * 10000 return self.CurrentDensity_value + rand_val + def getScreenCurrent(self): + rand_val = (random.random() - 0.5) * 10000 + return self.CurrentDensity_value + rand_val + + def isfocusscreenin(self) -> bool: + return False + def getBrightness(self) -> int: return self.Brightness_value def setBrightness(self, value: int): self.Brightness_value = value + def getBrightnessValue(self) -> int: + return self.Brightness_value + + def setBrightnessValue(self, value: int): + self.Brightness_value = value + def getMagnification(self) -> int: if self.getFunctionMode() == 'diff': return self.Magnification_value_diff @@ -356,6 +375,14 @@ def setGunShift(self, x: int, y: int): self.GunShift_x = x self.GunShift_y = y + def getBeamAlignShift(self) -> (float, float): + """get the Gun-Shift values.""" + return self.getGunShift() + + def setBeamAlignShift(self, x: float, y: float) -> None: + """set Gun-Shift values.""" + self.setGunShift(x, y) + def getGunTilt(self) -> Tuple[int, int]: return self.GunTilt_x, self.GunTilt_y @@ -377,6 +404,15 @@ def setBeamTilt(self, x: int, y: int): self.BeamTilt_x = x self.BeamTilt_y = y + def getDarkFieldTilt(self) -> (float, float): + """get the dark field tile value.""" + return self.IlluminationTiltX, self.IlluminationTiltX + + def setDarkFieldTilt(self, x: float, y: float) -> None: + """set the dark field tilt value.""" + self.IlluminationTiltX = x + self.IlluminationTiltY = y + def getImageShift1(self) -> Tuple[int, int]: return self.ImageShift1_x, self.ImageShift1_y @@ -391,10 +427,29 @@ def setImageShift2(self, x: int, y: int): self.ImageShift2_x = x self.ImageShift2_y = y + def getImageBeamShift(self): + return self.ImageBeamShift_x, self.ImageBeamShift_y + + def setImageBeamShift(self, x: float, y: float) -> None: + self.ImageBeamShift_x = x + self.ImageBeamShift_y = y + + def getHolderType(self) -> int: + return 0 + def getStagePosition(self) -> StagePositionTuple: return (self.StagePosition_x, self.StagePosition_y, self.StagePosition_z, self.StagePosition_a, self.StagePosition_b) + def isAThreadAlive(self) -> bool: + """Return goniotool status, always False.""" + return False + + def getStageSpeed(self) -> float: + """Return Stagespeed, can not be read on Tecnai = constant(0.5).""" + print('StageSpeed can not be read on Tecnai') + return 0.5 + def isStageMoving(self) -> bool: self.getStagePosition() # trigger update of self._is_moving # print(res, self._is_moving) @@ -468,7 +523,7 @@ def getRotationSpeed(self) -> int: def setRotationSpeed(self, value: int): self._stage_dict['a']['speed_setting'] = value - self._stage_dict['a']['speed'] = 10.0 * (value / 12) + self._stage_dict['a']['speed'] = value * 20 def getFunctionMode(self) -> str: """Mag1, mag2, lowmag, samag, diff.""" @@ -495,6 +550,30 @@ def setDiffFocus(self, value: int, confirm_mode: bool = True): raise TEMValueError("Must be in 'diff' mode to set DiffFocus") self.DiffractionFocus_value = value + def getDiffFocusValue(self, confirm_mode: bool = True) -> float: + """get the diffraction focus value.""" + if not self.getFunctionMode() == 'diff': + raise TEMValueError("Must be in 'diff' mode to get DiffFocus") + return self.DiffractionFocus_value + + def setDiffFocusValue(self, value: float, confirm_mode: bool = True) -> None: + """set the diffraction focus value.""" + if not self.getFunctionMode() == 'diff': + raise TEMValueError("Must be in 'diff' mode to set DiffFocus") + self.DiffractionFocus_value = value + + def getFocus(self) -> float: + """get the Defocus value.""" + if not self.getFunctionMode() in ['lowmag', 'mag1', 'samag', 'mag2']: + raise TEMValueError("Must be in 'mag' mode to get Focus") + return self.DiffractionFocus_value + + def setFocus(self, value: float) -> None: + """set the Defocus value.""" + if not self.getFunctionMode() in ['lowmag', 'mag1', 'samag', 'mag2']: + raise TEMValueError("Must be in 'mag' mode to set Focus") + self.DiffractionFocus_value = value + def setIntermediateLens1(self, value: int): """IL1.""" self.IntermediateLens1_value = value diff --git a/instamaticServer/TEMController/tecnai_microscope.py b/instamaticServer/TEMController/tecnai_microscope.py index 7d9767f..b51ca90 100644 --- a/instamaticServer/TEMController/tecnai_microscope.py +++ b/instamaticServer/TEMController/tecnai_microscope.py @@ -327,7 +327,6 @@ def getBeamAlignShift(self) -> (float, float): def setBeamAlignShift(self, x: float, y: float) -> None: """set Gun-Shift values.""" self.setGunShift(x, y) - ###Illumination def getSpotSize(self) -> int: diff --git a/tests.py b/tests.py index 61c4a46..7dc78c3 100644 --- a/tests.py +++ b/tests.py @@ -7,6 +7,7 @@ import unittest from typing import Any, Dict +from instamaticServer.TEMController.simu_microscope import SimuMicroscope from instamaticServer.utils.config import NS, config, dict_to_namespace from instamaticServer.tem_server import stop_program_event @@ -14,7 +15,7 @@ _conf = config() PRECISION_NM = 250 PRECISION_DEG = 0.1 -TIMEOUT = 5 +TIMEOUT = 30 class TestConfig(unittest.TestCase): @@ -139,7 +140,8 @@ def cam_send(self, func: str, args = (), kwargs = None): def test_20_getHolderType(self): r = self.tem_send('getHolderType') - self.assertIsInstance(r, self.const.StageHolderType) + if not isinstance(self.tem_server.device, SimuMicroscope): + self.assertIsInstance(r, self.const.StageHolderType) def test_21_getStagePosition(self): r = self.tem_send('getStagePosition') @@ -173,8 +175,8 @@ def test_30_setStagePosition(self): self.assertAlmostEqual(r[4], p[4], delta=PRECISION_DEG) def test_31_setStagePosition(self): - p = {'x': 10000, 'y': 10000} - self.tem_send('setStagePosition', kwargs=p) + self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) + self.tem_send('setStagePosition', kwargs={'x': 10000, 'y': 10000}) r = self.tem_send('getStagePosition') self.assertAlmostEqual(r[0], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[1], 10000, delta=PRECISION_NM) @@ -183,8 +185,8 @@ def test_31_setStagePosition(self): self.assertAlmostEqual(r[4], 0, delta=PRECISION_DEG) def test_32_setStagePosition(self): - p = {'z': 10000} - self.tem_send('setStagePosition', kwargs=p) + self.tem_send('setStagePosition', (10000, 10000, 0, 0, 0)) + self.tem_send('setStagePosition', kwargs={'z': 10000}) r = self.tem_send('getStagePosition') self.assertAlmostEqual(r[0], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[1], 10000, delta=PRECISION_NM) @@ -192,40 +194,41 @@ def test_32_setStagePosition(self): def test_33_setStagePosition(self): self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) - p = {'a': 10} - self.tem_send('setStagePosition', kwargs=p) + self.tem_send('setStagePosition', kwargs={'a': 10}) r = self.tem_send('getStagePosition') self.assertAlmostEqual(r[3], 10, delta=PRECISION_DEG) def test_35_setStagePosition(self): self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'x': 10000}) + self.tem_send('setStagePosition', kwargs={'x': 1000}) t1 = time.perf_counter() self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.1}) t2 = time.perf_counter() - self.assertLess(t1 - t0, t2 - t1) - self.tem_send('setStagePosition', kwargs={'x': 10000, 'speed': 0.05}) + self.tem_send('setStagePosition', kwargs={'x': 1000, 'speed': 0.05}) t3 = time.perf_counter() - self.assertLess(t2 - t1, t3 - t2) self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.02}) t4 = time.perf_counter() - self.assertLess(t3 - t2, t4 - t3) + if not isinstance(self.tem_server.device, SimuMicroscope): + self.assertLess(t1 - t0, t2 - t1) + self.assertLess(t2 - t1, t3 - t2) + self.assertLess(t3 - t2, t4 - t3) def test_36_setStagePosition(self): self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'a': 5}) + self.tem_send('setStagePosition', kwargs={'a': 2}) t1 = time.perf_counter() self.tem_send('setStagePosition', kwargs={'a': 0, 'speed': 0.1}) t2 = time.perf_counter() - self.assertLess(t1 - t0, t2 - t1) - self.tem_send('setStagePosition', kwargs={'a': 5, 'speed': 0.05}) + self.tem_send('setStagePosition', kwargs={'a': 2, 'speed': 0.05}) t3 = time.perf_counter() - self.assertLess(t2 - t1, t3 - t2) self.tem_send('setStagePosition', kwargs={'a': 0, 'speed': 0.02}) t4 = time.perf_counter() - self.assertLess(t3 - t2, t4 - t3) + if not isinstance(self.tem_server.device, SimuMicroscope): + self.assertLess(t2 - t1, t3 - t2) + self.assertLess(t1 - t0, t2 - t1) + self.assertLess(t3 - t2, t4 - t3) def test_38_setStagePosition(self): self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) @@ -260,13 +263,8 @@ def test_46_setRotationSpeed(self): self.tem_send('setStageA', kwargs={'value': 5}) t3 = time.perf_counter() self.assertLess(t2 - t1, t3 - t2) - self.tem_send('setRotationSpeed', (0.02, )) - self.tem_send('setStageA', kwargs={'value': 0}) - t4 = time.perf_counter() - self.assertLess(t3 - t2, t4 - t3) - s = self.tem_send('setRotationSpeed') - self.assertEqual(s, 0.02) self.tem_send('setRotationSpeed', (1.0, )) + self.tem_send('setStageA', kwargs={'value': 0}) def test_48_setStageA(self): self.tem_send('setStageA', (0, )) @@ -322,23 +320,27 @@ def test_63_getScreenPosition(self): self.assertIn(r, {'up', 'down', ''}) def test_64_getDiffFocus(self): + func_mode = self.tem_send('getFunctionMode') + self.tem_send('setFunctionMode', ('diff', )) r = self.tem_send('getDiffFocus') + self.tem_send('setFunctionMode', (func_mode,)) self.assertGreater(r, 0) self.assertLess(r, 65536) def test_65_getDiffFocusValue(self): func_mode = self.tem_send('getFunctionMode') - self.tem_send('setFunctionMode', 'diff') + self.tem_send('setFunctionMode', ('diff', )) r = self.tem_send('getDiffFocusValue') - print(func_mode) self.tem_send('setFunctionMode', (func_mode, )) - self.assertGreater(r, -1.0) - self.assertLess(r, 1.0) + if not isinstance(self.tem_server.device, SimuMicroscope): + self.assertGreater(r, -1.0) + self.assertLess(r, 1.0) def test_65_getFocus(self): r = self.tem_send('getFocus') - self.assertGreater(r, -1.0) - self.assertLess(r, 1.0) + if not isinstance(self.tem_server.device, SimuMicroscope): + self.assertGreater(r, -1.0) + self.assertLess(r, 1.0) def test_67_FunctionMode(self): r = self.tem_send('getFunctionMode') @@ -357,7 +359,8 @@ def test_69_MagnificationIndex(self): def test_70_getDarkFieldTilt(self): r = self.tem_send('getDarkFieldTilt') - self.assertIsInstance(r[0], float) + self.assertIsInstance(r[0], (int, float)) + self.assertIsInstance(r[1], (int, float)) def test_71_getImageShift1(self): r = self.tem_send('getImageShift1') From 268d8b8243c0140f7f922e13a59e00c2da0b619a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 3 Dec 2025 12:41:00 +0100 Subject: [PATCH 06/26] Try removing remaining Python 3.4 incompatibilities --- instamaticServer/TEMController/simu_camera.py | 2 +- tests.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/instamaticServer/TEMController/simu_camera.py b/instamaticServer/TEMController/simu_camera.py index 440eb14..3ddcf54 100644 --- a/instamaticServer/TEMController/simu_camera.py +++ b/instamaticServer/TEMController/simu_camera.py @@ -37,7 +37,7 @@ def __init__(self, name='simulate'): self.name = name self.load_defaults() self.establish_connection() - logger.info(f'Camera simulate initialized') + logger.info('Camera simulate initialized') atexit.register(self.release_connection) def __enter__(self): diff --git a/tests.py b/tests.py index 7dc78c3..43f3181 100644 --- a/tests.py +++ b/tests.py @@ -5,7 +5,6 @@ import threading import time import unittest -from typing import Any, Dict from instamaticServer.TEMController.simu_microscope import SimuMicroscope from instamaticServer.utils.config import NS, config, dict_to_namespace @@ -55,7 +54,7 @@ def test_msgpack_serializer(self): class TestServer(unittest.TestCase): @classmethod - def setUpClass(cls) -> None: + def setUpClass(cls): from instamaticServer.tem_server import CamServer, TemServer, listen cls.tem_server = TemServer() cls.tem_server.start() @@ -95,16 +94,16 @@ def setUpClass(cls) -> None: # atexit.register(cls.socket_cam.close) @classmethod - def tearDownClass(cls) -> None: + def tearDownClass(cls): stop_program_event.set() for thread in cls.threads: thread.join(timeout=5) if thread.is_alive(): - print(f"Thread {thread.name} did not exit") + print('Thread %s did not exit' % thread.name) cls.socket_tem.close() @staticmethod - def socket_send(s: socket.socket, func: str, args = (), kwargs = None): + def socket_send(s, func, args = (), kwargs = None): from instamaticServer.serializer import dumper, loader from instamaticServer.utils.exceptions import TEMCommunicationError, exception_list kwargs = kwargs or {} @@ -112,13 +111,13 @@ def socket_send(s: socket.socket, func: str, args = (), kwargs = None): buffer_size = 1024 if func in ('get_image', 'get_movie'): buffer_size += 8 * _conf.camera.dimensions[0] * _conf.camera.dimensions[1] - s.send(dumper(d)) + s.sendall(dumper(d)) response = s.recv(buffer_size) if response: for _ in range(10): # warrants all image/movie is collected try: status, data = loader(response) - except (pickle.UnpicklingError, json.JSONDecodeError, RuntimeError): + except (pickle.UnpicklingError, ValueError, RuntimeError): response += s.recv(buffer_size) else: break @@ -132,10 +131,10 @@ def socket_send(s: socket.socket, func: str, args = (), kwargs = None): else: raise ConnectionError('Unknown status code: %s' % status) - def tem_send(self, func: str, args = (), kwargs = None): + def tem_send(self, func, args = (), kwargs = None): return self.socket_send(self.socket_tem, func, args, kwargs) - def cam_send(self, func: str, args = (), kwargs = None): + def cam_send(self, func, args = (), kwargs = None): return self.socket_send(self.socket_cam, func, args, kwargs) def test_20_getHolderType(self): From f276cb3a10a6db4e451b4085cd8566a52e947e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 3 Dec 2025 13:48:52 +0100 Subject: [PATCH 07/26] Fix tests to do logging & printing neatly --- instamaticServer/TEMController/simu_camera.py | 2 +- .../TEMController/simu_microscope.py | 15 +++++--- .../TEMController/tecnai_camera.py | 4 +-- .../TEMController/tecnai_microscope.py | 36 +++++++------------ instamaticServer/tem_server.py | 25 +++++++++---- tests.py | 2 -- 6 files changed, 43 insertions(+), 41 deletions(-) diff --git a/instamaticServer/TEMController/simu_camera.py b/instamaticServer/TEMController/simu_camera.py index 3ddcf54..64e36a4 100644 --- a/instamaticServer/TEMController/simu_camera.py +++ b/instamaticServer/TEMController/simu_camera.py @@ -14,7 +14,7 @@ np = False -logger = logging.getLogger(__name__) +logger = logging.getLogger('cam') class SimuCamera(metaclass=Singleton): diff --git a/instamaticServer/TEMController/simu_microscope.py b/instamaticServer/TEMController/simu_microscope.py index 0db98f8..1f2832d 100644 --- a/instamaticServer/TEMController/simu_microscope.py +++ b/instamaticServer/TEMController/simu_microscope.py @@ -1,3 +1,4 @@ +import logging import random import time from typing import Optional, Tuple, Union @@ -7,6 +8,9 @@ from instamaticServer.utils.types import StagePositionTuple, float_deg, int_nm +logger = logging.getLogger('tem') + + NTRLMAPPING = { 'GUN1': 0, 'GUN2': 1, @@ -153,6 +157,8 @@ def __init__(self, name: str = None): 't0': 0.0, } + logger.info('Microscope simulate initialized') + ##self.goniotool_available = config.settings.use_goniotool self.goniotool_available = False ##auf Klasse GonioToolClient Obacht geben @@ -161,8 +167,8 @@ def __init__(self, name: str = None): try: self.goniotool = GonioToolClient() except Exception as e: - print('GonioToolClient:', e) - print('Could not connect to GonioToolServer, goniotool unavailable!') + logger.warning('GonioToolClient:', e) + logger.warning('Could not connect to GonioToolServer, goniotool unavailable!') self.goniotool_available = False #config.settings.use_goniotool = False @@ -447,12 +453,11 @@ def isAThreadAlive(self) -> bool: def getStageSpeed(self) -> float: """Return Stagespeed, can not be read on Tecnai = constant(0.5).""" - print('StageSpeed can not be read on Tecnai') + logger.info('StageSpeed can not be read on Tecnai') return 0.5 def isStageMoving(self) -> bool: self.getStagePosition() # trigger update of self._is_moving - # print(res, self._is_moving) return self._is_moving def waitForStage(self, delay: float = 0.1): @@ -590,7 +595,7 @@ def setDiffShift(self, x: int, y: int): self.DiffractionShift_y = y def release_connection(self): - print('Connection to microscope released') + logger.info('Connection to microscope released') def isBeamBlanked(self) -> bool: return self.beamblank diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index e19eddf..735a10e 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -14,7 +14,7 @@ np = False -logger = logging.getLogger(__name__) +logger = logging.getLogger('cam') class TecnaiCamera(metaclass=Singleton): @@ -37,7 +37,7 @@ def __init__(self, name='tecnai'): self.name = name self.load_defaults() self.acq, self.cam = self.establish_connection() - logger.info(f'Camera Tecnai initialized') + logger.info(f'Camera Tecnai connection established') atexit.register(self.release_connection) def __enter__(self): diff --git a/instamaticServer/TEMController/tecnai_microscope.py b/instamaticServer/TEMController/tecnai_microscope.py index b51ca90..057d0fb 100644 --- a/instamaticServer/TEMController/tecnai_microscope.py +++ b/instamaticServer/TEMController/tecnai_microscope.py @@ -12,18 +12,9 @@ from instamaticServer.utils.types import StagePositionTuple, float_deg, int_nm -_FUNCTION_MODES = {1: 'lowmag', 2: 'mag1', 3: 'samag', 4: 'mag2', 5: 'LAD', 6: 'diff'} - -#diff=D, LAD=LAD, lowmag=LM, mag1=Mi, samag=SA, mag2=Mh in Functionmodes - +logger = logging.getLogger('tem') - -#dict([('D', [0.0265, 0.035, 0.044, 0.062, 0.071, 0.089, 0.135, 0.175, 0.265, 0.43, 0.6, 0.86, 1.65, 2.65, 3.5, 4.1]), -# ('LAD', [4.5, 7.1, 9, 12.5, 18, 27, 36, 53, 71, 81, 130, 180, 245, 360, 530, 720, 790, 810, 960, 1100, 1300]), -# ('LM', [19, 25, 35, 50, 65, 82, 105, 145, 200, 300, 390, 500, 730, 980, 1350, 1850]), -# ('Mi', [2250, 3500, 4400]), -# ('SA', [6200, 8700, 13500, 17000, 26000, 34000, 38000, 63000, 86000, 125000, 175000, 250000, 350000, 400000]), -# ('Mh', [440000, 520000, 610000, 700000, 780000, 910000])]) +_FUNCTION_MODES = {1: 'lowmag', 2: 'mag1', 3: 'samag', 4: 'mag2', 5: 'LAD', 6: 'diff'} class TecnaiMicroscope(metaclass=Singleton): @@ -36,11 +27,9 @@ def __init__(self, name: str=None) -> None: except: raise - print('FEI Scripting initializing...') - ## TEM interfaces the GUN, stage etc + logger.info('FEI Scripting initializing...') + ## TEM interfaces the GUN, stage etc. + enum constants self._tem = comtypes.client.CreateObject('TEMScripting.Instrument', comtypes.CLSCTX_ALL) - - ## TEM enum constants self._tem_constant = comtypes.client.Constants(self._tem) self._t = 0 @@ -51,12 +40,11 @@ def __init__(self, name: str=None) -> None: time.sleep(1) self._t += 1 if self._t > 3: - print('Waiting for microscope, t = %ss' % self._t) + logger.info('Waiting for microscope, t = %ss' % self._t) if self._t > 30: raise TEMCommunicationError('Cannot establish microscope connection (timeout).') - self._logger = logging.getLogger(__name__) - self._logger.info('Microscope connection established') + logger.info('Microscope Tecnai connection established') #close the network connection atexit.register(TecnaiMicroscope.release_connection) @@ -88,7 +76,7 @@ def getStagePosition(self) -> StagePositionTuple: def getStageSpeed(self) -> float: """Return Stagespeed, can not be read on Tecnai = constant(0.5).""" - print('StageSpeed can not be read on Tecnai') + logger.info('StageSpeed can not be read on Tecnai') return 0.5 def is_goniotool_available(self) -> bool: @@ -240,11 +228,11 @@ def waitForStage(self, delay: float=0.1) -> None: def setStageSpeed(self, value: float) -> None: """Set Stage speed, not available on Tecnai.""" - print('StageSpeed can not be set on Tecnai') + logger.info('StageSpeed can not be set on Tecnai') def stopStage(self) -> None: """Stop Stage, not available on Tecnai.""" - print('stopStage: not available on Tecnai.') + logger.info('stopStage: not available on Tecnai.') def setRotationSpeed(self, value: float) -> None: """Set rotationspeed of the alpha rotation.""" @@ -402,7 +390,7 @@ def setCondensorLensStigmator(self, x: float, y: float) -> None: ###Projection def getCurrentDensity(self) -> float: """Get the current density, not available on Tecnai.""" - print('getCurrentDensity: not available on the Tecnai.') + logger.info('getCurrentDensity: not available on the Tecnai.') return 0 def getScreenCurrent(self) -> float: @@ -669,11 +657,11 @@ def setIntermediateLensStigmator(self, x: float, y: float) -> None: def release_connection() -> None: """release the COM-connection.""" comtypes.CoUninitialize() - print('Connection to microscope released') + logger.info('Connection to microscope released') def getApertureSize(self, aperture: str) -> None: """not available on Tecnai.""" - print('getApertureSize, not available on Tecnai.') + logger.info('getApertureSize, not available on Tecnai.') if __name__ == '__main__': diff --git a/instamaticServer/tem_server.py b/instamaticServer/tem_server.py index 70aa666..5295092 100644 --- a/instamaticServer/tem_server.py +++ b/instamaticServer/tem_server.py @@ -2,6 +2,7 @@ import logging import queue import socket +import sys import threading import time import traceback @@ -22,7 +23,18 @@ logfile = 'tem_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' -logging.basicConfig(level=logging.INFO, filename='tem_server.log', format=logging_fmt) +logging.basicConfig(level=15, filename='tem_server.log', format=logging_fmt) +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.setFormatter(logging.Formatter(logging_fmt)) +logging.getLogger().addHandler(stdout_handler) +logging.addLevelName(15, "EVAL") + +def log_eval(logger, status, func_name, args, kwargs, ret): + if logger.isEnabledFor(15): + args_list = [repr(a) for a in args] + args_list += ['%s=%r' % (k, v) for k, v in kwargs.items()] + args_str = ', '.join(args_list) + logger.log(15, '%s | %s(%s): %s', status, func_name, args_str, ret) class DeviceServer(threading.Thread): @@ -78,14 +90,13 @@ def run(self) -> None: status = 500 self.responses.put((status, ret)) - self.logger.info("%s | %s %s: %s", now, status, func_name, ret) + log_eval(self.logger, status, func_name, args, kwargs, ret) - self.logger.info('Terminating %s %s server thread', self.device_kind, self._name) + self.logger.info('Terminating %s %s server thread', self.device_kind, self.device.name) def evaluate(self, func_name: str, args: list, kwargs: dict) -> Any: - """Evaluate the function `func_name` on `self.device` and call it with - `args` and `kwargs`.""" - self.logger.debug('eval %s %s %s', func_name, args, kwargs) + """Eval function `func_name` on `self.device` with `args` & `kwargs`.""" + self.logger.debug('evaluate(func_name=%s, args=%s, kwargs=%s)', func_name, args, kwargs) f = getattr(self.device, func_name) return f(*args, **kwargs) if callable(f) else f @@ -127,7 +138,7 @@ class CamServer(DeviceServer): def __init__(self, name=None) -> None: super(CamServer, self).__init__(name=name) - self.logger.setLevel(logging.WARNING) + self.logger.setLevel(logging.INFO) def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: diff --git a/tests.py b/tests.py index 43f3181..7091ff4 100644 --- a/tests.py +++ b/tests.py @@ -1,5 +1,3 @@ -import atexit -import json import pickle import socket import threading From b5d06652b20a2e23ad516e1f058bc502b13ca3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 3 Dec 2025 15:49:22 +0100 Subject: [PATCH 08/26] Add atexit to be sure `TestServer` exits correctly --- instamaticServer/tem_server.py | 10 ++++++++-- tests.py | 25 +++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/instamaticServer/tem_server.py b/instamaticServer/tem_server.py index 5295092..1e03211 100644 --- a/instamaticServer/tem_server.py +++ b/instamaticServer/tem_server.py @@ -21,11 +21,18 @@ BUFSIZE = 1024 TIMEOUT = 0.5 + +class MicrosecondFormatter(logging.Formatter): + def formatTime(self, record, datefmt=None): + t = datetime.datetime.fromtimestamp(record.created) + return t.strftime(datefmt) if datefmt else t.strftime("%H:%M:%S.%f") + + logfile = 'tem_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' logging.basicConfig(level=15, filename='tem_server.log', format=logging_fmt) stdout_handler = logging.StreamHandler(sys.stdout) -stdout_handler.setFormatter(logging.Formatter(logging_fmt)) +stdout_handler.setFormatter(MicrosecondFormatter(logging_fmt)) logging.getLogger().addHandler(stdout_handler) logging.addLevelName(15, "EVAL") @@ -68,7 +75,6 @@ def run(self) -> None: self.logger.info('Initialized %s %s server thread', self.device_kind, self.device.name) while True: - now = datetime.datetime.now().strftime('%H:%M:%S.%f') try: cmd = self.requests.get(timeout=TIMEOUT) except queue.Empty: diff --git a/tests.py b/tests.py index 7091ff4..92a5688 100644 --- a/tests.py +++ b/tests.py @@ -1,3 +1,4 @@ +import atexit import pickle import socket import threading @@ -59,6 +60,7 @@ def setUpClass(cls): cls.tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') cls.tem_listener.start() cls.threads = [cls.tem_server, cls.tem_listener] + atexit.register(cls.tearDownClass) t0 = time.perf_counter() while getattr(cls.tem_server, 'device', None) is None: @@ -82,23 +84,38 @@ def setUpClass(cls): cls.socket_tem = socket.socket(socket.AF_INET, socket.SOCK_STREAM) cls.socket_tem.connect((tem_host, tem_port)) cls.socket_tem.settimeout(TIMEOUT) - # atexit.register(cls.socket_tem.close) cam_host = _conf.default_settings['cam_server_host'] cam_port = _conf.default_settings['cam_server_port'] cls.socket_cam = socket.socket(socket.AF_INET, socket.SOCK_STREAM) cls.socket_cam.connect((cam_host, cam_port)) cls.socket_cam.settimeout(TIMEOUT) - # atexit.register(cls.socket_cam.close) @classmethod def tearDownClass(cls): + print('TEARDOWN???') stop_program_event.set() - for thread in cls.threads: + while cls.threads: + thread = cls.threads.pop(0) thread.join(timeout=5) if thread.is_alive(): print('Thread %s did not exit' % thread.name) - cls.socket_tem.close() + + if hasattr(cls, 'socket_tem') and cls.socket_tem: + try: + cls.socket_tem.close() + except Exception as e: + print("Error closing socket_tem:", e) + finally: + cls.socket_tem = None + + if hasattr(cls, 'socket_cam') and cls.socket_cam: + try: + cls.socket_cam.close() + except Exception as e: + print("Error closing socket_cam:", e) + finally: + cls.socket_cam = None @staticmethod def socket_send(s, func, args = (), kwargs = None): From 2284ae356502156df6fa66bc0d52809e2450595f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 3 Dec 2025 19:27:19 +0100 Subject: [PATCH 09/26] =?UTF-8?q?Run=20tests=20on=20tecnai=20=E2=80=93=20s?= =?UTF-8?q?uccess,=20new=20interface=20works!?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- instamaticServer/TEMController/simu_camera.py | 5 +- .../tem_server.py => tem_server.py | 550 +++++++++--------- tests.py | 31 +- 3 files changed, 298 insertions(+), 288 deletions(-) rename instamaticServer/tem_server.py => tem_server.py (97%) diff --git a/instamaticServer/TEMController/simu_camera.py b/instamaticServer/TEMController/simu_camera.py index 64e36a4..04f7b8a 100644 --- a/instamaticServer/TEMController/simu_camera.py +++ b/instamaticServer/TEMController/simu_camera.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import atexit import logging import time @@ -72,8 +70,7 @@ def get_image(self, exposure: Optional[float] = None, binsize: int = 1): img = 256 * np.random.random_sample((dx, dy)) else: import random - v = list(range(0, 256)) - img = [random.sample(v, dx) for _ in range(dy)] + img = [[random.randint(0, 255) for _ in range(dx)] for _ in range(dy)] while time.perf_counter() - t0 < exposure: time.sleep(0.001) return img diff --git a/instamaticServer/tem_server.py b/tem_server.py similarity index 97% rename from instamaticServer/tem_server.py rename to tem_server.py index 1e03211..d64a1fa 100644 --- a/instamaticServer/tem_server.py +++ b/tem_server.py @@ -1,275 +1,275 @@ -import datetime -import logging -import queue -import socket -import sys -import threading -import time -import traceback - -from typing import Any, Type, Callable - -from instamaticServer.TEMController.camera import get_camera -from instamaticServer.TEMController.microscope import get_microscope -from instamaticServer.serializer import dumper, loader -from instamaticServer.utils.config import config - - -stop_program_event = threading.Event() - -_conf = config() -BUFSIZE = 1024 -TIMEOUT = 0.5 - - -class MicrosecondFormatter(logging.Formatter): - def formatTime(self, record, datefmt=None): - t = datetime.datetime.fromtimestamp(record.created) - return t.strftime(datefmt) if datefmt else t.strftime("%H:%M:%S.%f") - - -logfile = 'tem_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') -logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' -logging.basicConfig(level=15, filename='tem_server.log', format=logging_fmt) -stdout_handler = logging.StreamHandler(sys.stdout) -stdout_handler.setFormatter(MicrosecondFormatter(logging_fmt)) -logging.getLogger().addHandler(stdout_handler) -logging.addLevelName(15, "EVAL") - -def log_eval(logger, status, func_name, args, kwargs, ret): - if logger.isEnabledFor(15): - args_list = [repr(a) for a in args] - args_list += ['%s=%r' % (k, v) for k, v in kwargs.items()] - args_str = ', '.join(args_list) - logger.log(15, '%s | %s(%s): %s', status, func_name, args_str, ret) - - -class DeviceServer(threading.Thread): - """General microscope / camera (Acquisition) communication server. - - Takes a `name` of the microscope/camera and initializes appropriate device. - When `TemServer.start` thread method is called, `TemServer.run` starts. - The server will wait for cmd `requests` to appear in the queue, evaluate - them in order, and return the result via each client's `response_queue`. - """ - - device_abbr = None # type: str - device_kind = None # type: str - device_getter = None # type: Callable - requests = None # type: queue.Queue - responses = None # type: queue.Queue - host = 'localhost' # type: str - port = None # type: int - - def __init__(self, name = None) -> None: - super().__init__(name=self.device_kind + '_server') - self.interface_name = name - self.logger = logging.getLogger(self.device_abbr + 'S') # temS/camS server - self.device = None - self.verbose = False - - def run(self) -> None: - """Start the server thread.""" - self.device = self.device_getter(name=self.interface_name) - self.device.get_attrs = self.get_attrs - self.logger.info('Initialized %s %s server thread', self.device_kind, self.device.name) - - while True: - try: - cmd = self.requests.get(timeout=TIMEOUT) - except queue.Empty: - if stop_program_event.is_set(): - break - continue - - func_name = cmd.get('func_name', cmd.get('attr_name')) - args = cmd.get('args', ()) - kwargs = cmd.get('kwargs', {}) - - try: - ret = self.evaluate(func_name, args, kwargs) - status = 200 - except Exception as e: - traceback.print_exc() - self.logger.exception(e) - ret = (e.__class__.__name__, e.args) - status = 500 - - self.responses.put((status, ret)) - log_eval(self.logger, status, func_name, args, kwargs, ret) - - self.logger.info('Terminating %s %s server thread', self.device_kind, self.device.name) - - def evaluate(self, func_name: str, args: list, kwargs: dict) -> Any: - """Eval function `func_name` on `self.device` with `args` & `kwargs`.""" - self.logger.debug('evaluate(func_name=%s, args=%s, kwargs=%s)', func_name, args, kwargs) - f = getattr(self.device, func_name) - return f(*args, **kwargs) if callable(f) else f - - def get_attrs(self): - """Get attributes from cam object to update __dict__ on client side.""" - attrs = {} - for item in dir(self.device): - if item.startswith('_'): - continue - obj = getattr(self.device, item) - if not callable(obj): - attrs[item] = type(obj) - - return attrs - - -class TemServer(DeviceServer): - """TEM communcation server.""" - - device_abbr = 'tem' - device_kind = 'microscope' - device_getter = staticmethod(get_microscope) - requests = queue.Queue(maxsize=1) - responses = queue.Queue(maxsize=1) - host = _conf.default_settings['tem_server_host'] - port = _conf.default_settings['tem_server_port'] - - -class CamServer(DeviceServer): - """FEI Tecnai/Titan Acquisition camera communication server.""" - - device_abbr = 'cam' - device_kind = 'camera' - device_getter = staticmethod(get_camera) - requests = queue.Queue(maxsize=1) - responses = queue.Queue(maxsize=1) - host = _conf.default_settings['cam_server_host'] - port = _conf.default_settings['cam_server_port'] - - def __init__(self, name=None) -> None: - super(CamServer, self).__init__(name=name) - self.logger.setLevel(logging.INFO) - - -def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: - """Handle incoming connection, put command on the Queue `q`, which is then - handled by TEMServer.""" - with conn: - conn.settimeout(TIMEOUT) - while True: - if stop_program_event.is_set(): - break - - try: - data = conn.recv(BUFSIZE) - except socket.timeout: - continue - - if not data: - break - - data = loader(data) - - if data == 'exit' or data == 'kill': - break - - server_type.requests.put(data) - response = server_type.responses.get() - serialized = dumper(response) - conn.sendall(serialized) - - -def listen(server_type: Type[DeviceServer]) -> None: - """Listen on a given server host/port and handle incoming instructions""" - - logger = logging.getLogger(server_type.device_abbr + 'L') # temL/camL listener - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as device_client: - device_client.bind((server_type.host, server_type.port)) - device_client.settimeout(TIMEOUT) - device_client.listen(1) - logger.info('Server listening on %s:%s', server_type.host, server_type.port) - while True: - if stop_program_event.is_set(): - break - try: - connection, _ = device_client.accept() - handle(connection, server_type) - except socket.timeout: - pass - except Exception as e: - logger.exception('Exception when handling connection: %s', e) - logger.info('Terminating %s listener thread', server_type.device_kind) - - -def main() -> None: - """ - Connects to the TEM and starts a server for microscope communication. - Opens a socket on port {HOST}:{PORT}. - - This program initializes a connection to the TEM as defined in the config. - The purpose of this program is to isolate the microscope connection in - a separate process for improved stability of the interface in case - instamatic crashes or is started and stopped frequently. - For running the GUI, the temserver is required. - Another reason is that it allows for remote connections from different PCs. - The connection goes over a TCP socket. - - The host and port are defined in `config/settings.yaml`. - - The data sent over the socket is a serialized dictionary with the following: - - - `func_name`: Name of the function to call (str) - - `args`: (Optional) List of arguments for the function (list) - - `kwargs`: (Optiona) Dictionary of keyword arguments for the function (dict) - - The response is returned as a serialized object. - """ - - import argparse - - parser = argparse.ArgumentParser(description=main.__doc__) - parser.add_argument('-t', '--microscope', action='store', - help='Override microscope to use.') - parser.add_argument('-c', '--camera', action='store_true', - help='If selected, start separate threads for a camera') - - parser.set_defaults(microscope=None) - options = parser.parse_args() - - logging.info('Tecnai server starting') - - tem_server = TemServer(name=options.microscope) - tem_server.start() - - tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') - tem_listener.start() - - threads = [tem_server, tem_listener] - - if options.camera: - logging.info('Waiting for the TEM singleton to initialize') - for _ in range(100): - if getattr(tem_server, 'device') is not None: # wait until TEM initialized - break - time.sleep(0.05) - else: # necessary check, Error extremely unlikely, TEM typically starts in ms - raise RuntimeError('Could not start TEM device on server in 5 seconds') - - cam_server = CamServer(name=options.microscope) - cam_server.start() - - cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') - cam_listener.start() - - threads.extend([cam_server, cam_listener]) - - try: - while not stop_program_event.is_set(): time.sleep(TIMEOUT) - except KeyboardInterrupt: - logging.info("Received KeyboardInterrupt, shutting down...") - finally: - stop_program_event.set() - for thread in threads: - thread.join() - logging.info('Tecnai server terminating') - logging.shutdown() - - -if __name__ == '__main__': - main() +import datetime +import logging +import queue +import socket +import sys +import threading +import time +import traceback + +from typing import Any, Type, Callable + +from instamaticServer.TEMController.camera import get_camera +from instamaticServer.TEMController.microscope import get_microscope +from instamaticServer.serializer import dumper, loader +from instamaticServer.utils.config import config + + +stop_program_event = threading.Event() + +_conf = config() +BUFSIZE = 1024 +TIMEOUT = 0.5 + + +class MicrosecondFormatter(logging.Formatter): + def formatTime(self, record, datefmt=None): + t = datetime.datetime.fromtimestamp(record.created) + return t.strftime(datefmt) if datefmt else t.strftime("%H:%M:%S.%f") + + +logfile = 'tem_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') +logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' +logging.basicConfig(level=15, filename='tem_server.log', format=logging_fmt) +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.setFormatter(MicrosecondFormatter(logging_fmt)) +logging.getLogger().addHandler(stdout_handler) +logging.addLevelName(15, "EVAL") + +def log_eval(logger, status, func_name, args, kwargs, ret): + if logger.isEnabledFor(15): + args_list = [repr(a) for a in args] + args_list += ['%s=%r' % (k, v) for k, v in kwargs.items()] + args_str = ', '.join(args_list) + logger.log(15, '%s | %s(%s): %s', status, func_name, args_str, ret) + + +class DeviceServer(threading.Thread): + """General microscope / camera (Acquisition) communication server. + + Takes a `name` of the microscope/camera and initializes appropriate device. + When `TemServer.start` thread method is called, `TemServer.run` starts. + The server will wait for cmd `requests` to appear in the queue, evaluate + them in order, and return the result via each client's `response_queue`. + """ + + device_abbr = None # type: str + device_kind = None # type: str + device_getter = None # type: Callable + requests = None # type: queue.Queue + responses = None # type: queue.Queue + host = 'localhost' # type: str + port = None # type: int + + def __init__(self, name = None) -> None: + super().__init__(name=self.device_kind + '_server') + self.interface_name = name + self.logger = logging.getLogger(self.device_abbr + 'S') # temS/camS server + self.device = None + self.verbose = False + + def run(self) -> None: + """Start the server thread.""" + self.device = self.device_getter(name=self.interface_name) + self.device.get_attrs = self.get_attrs + self.logger.info('Initialized %s %s server thread', self.device_kind, self.device.name) + + while True: + try: + cmd = self.requests.get(timeout=TIMEOUT) + except queue.Empty: + if stop_program_event.is_set(): + break + continue + + func_name = cmd.get('func_name', cmd.get('attr_name')) + args = cmd.get('args', ()) + kwargs = cmd.get('kwargs', {}) + + try: + ret = self.evaluate(func_name, args, kwargs) + status = 200 + except Exception as e: + traceback.print_exc() + self.logger.exception(e) + ret = (e.__class__.__name__, e.args) + status = 500 + + self.responses.put((status, ret)) + log_eval(self.logger, status, func_name, args, kwargs, ret) + + self.logger.info('Terminating %s %s server thread', self.device_kind, self.device.name) + + def evaluate(self, func_name: str, args: list, kwargs: dict) -> Any: + """Eval function `func_name` on `self.device` with `args` & `kwargs`.""" + self.logger.debug('evaluate(func_name=%s, args=%s, kwargs=%s)', func_name, args, kwargs) + f = getattr(self.device, func_name) + return f(*args, **kwargs) if callable(f) else f + + def get_attrs(self): + """Get attributes from cam object to update __dict__ on client side.""" + attrs = {} + for item in dir(self.device): + if item.startswith('_'): + continue + obj = getattr(self.device, item) + if not callable(obj): + attrs[item] = type(obj) + + return attrs + + +class TemServer(DeviceServer): + """TEM communcation server.""" + + device_abbr = 'tem' + device_kind = 'microscope' + device_getter = staticmethod(get_microscope) + requests = queue.Queue(maxsize=1) + responses = queue.Queue(maxsize=1) + host = _conf.default_settings['tem_server_host'] + port = _conf.default_settings['tem_server_port'] + + +class CamServer(DeviceServer): + """FEI Tecnai/Titan Acquisition camera communication server.""" + + device_abbr = 'cam' + device_kind = 'camera' + device_getter = staticmethod(get_camera) + requests = queue.Queue(maxsize=1) + responses = queue.Queue(maxsize=1) + host = _conf.default_settings['cam_server_host'] + port = _conf.default_settings['cam_server_port'] + + def __init__(self, name=None) -> None: + super(CamServer, self).__init__(name=name) + self.logger.setLevel(logging.INFO) + + +def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: + """Handle incoming connection, put command on the Queue `q`, which is then + handled by TEMServer.""" + with conn: + conn.settimeout(TIMEOUT) + while True: + if stop_program_event.is_set(): + break + + try: + data = conn.recv(BUFSIZE) + except socket.timeout: + continue + + if not data: + break + + data = loader(data) + + if data == 'exit' or data == 'kill': + break + + server_type.requests.put(data) + response = server_type.responses.get() + serialized = dumper(response) + conn.sendall(serialized) + + +def listen(server_type: Type[DeviceServer]) -> None: + """Listen on a given server host/port and handle incoming instructions""" + + logger = logging.getLogger(server_type.device_abbr + 'L') # temL/camL listener + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as device_client: + device_client.bind((server_type.host, server_type.port)) + device_client.settimeout(TIMEOUT) + device_client.listen(1) + logger.info('Server listening on %s:%s', server_type.host, server_type.port) + while True: + if stop_program_event.is_set(): + break + try: + connection, _ = device_client.accept() + handle(connection, server_type) + except socket.timeout: + pass + except Exception as e: + logger.exception('Exception when handling connection: %s', e) + logger.info('Terminating %s listener thread', server_type.device_kind) + + +def main() -> None: + """ + Connects to the TEM and starts a server for microscope communication. + Opens a socket on port {HOST}:{PORT}. + + This program initializes a connection to the TEM as defined in the config. + The purpose of this program is to isolate the microscope connection in + a separate process for improved stability of the interface in case + instamatic crashes or is started and stopped frequently. + For running the GUI, the temserver is required. + Another reason is that it allows for remote connections from different PCs. + The connection goes over a TCP socket. + + The host and port are defined in `config/settings.yaml`. + + The data sent over the socket is a serialized dictionary with the following: + + - `func_name`: Name of the function to call (str) + - `args`: (Optional) List of arguments for the function (list) + - `kwargs`: (Optiona) Dictionary of keyword arguments for the function (dict) + + The response is returned as a serialized object. + """ + + import argparse + + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument('-t', '--microscope', action='store', + help='Override microscope to use.') + parser.add_argument('-c', '--camera', action='store_true', + help='If selected, start separate threads for a camera') + + parser.set_defaults(microscope=None) + options = parser.parse_args() + + logging.info('Tecnai server starting') + + tem_server = TemServer(name=options.microscope) + tem_server.start() + + tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') + tem_listener.start() + + threads = [tem_server, tem_listener] + + if options.camera: + logging.info('Waiting for the TEM singleton to initialize') + for _ in range(100): + if getattr(tem_server, 'device') is not None: # wait until TEM initialized + break + time.sleep(0.05) + else: # necessary check, Error extremely unlikely, TEM typically starts in ms + raise RuntimeError('Could not start TEM device on server in 5 seconds') + + cam_server = CamServer(name=options.microscope) + cam_server.start() + + cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') + cam_listener.start() + + threads.extend([cam_server, cam_listener]) + + try: + while not stop_program_event.is_set(): time.sleep(TIMEOUT) + except KeyboardInterrupt: + logging.info("Received KeyboardInterrupt, shutting down...") + finally: + stop_program_event.set() + for thread in threads: + thread.join() + logging.info('Tecnai server terminating') + logging.shutdown() + + +if __name__ == '__main__': + main() diff --git a/tests.py b/tests.py index 92a5688..a1ec6cb 100644 --- a/tests.py +++ b/tests.py @@ -7,7 +7,7 @@ from instamaticServer.TEMController.simu_microscope import SimuMicroscope from instamaticServer.utils.config import NS, config, dict_to_namespace -from instamaticServer.tem_server import stop_program_event +from tem_server import stop_program_event _conf_dict = {'a': 1, 'b': {'c': 3, 'd': 4}} _conf = config() @@ -54,7 +54,7 @@ class TestServer(unittest.TestCase): @classmethod def setUpClass(cls): - from instamaticServer.tem_server import CamServer, TemServer, listen + from tem_server import CamServer, TemServer, listen cls.tem_server = TemServer() cls.tem_server.start() cls.tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') @@ -93,7 +93,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - print('TEARDOWN???') stop_program_event.set() while cls.threads: thread = cls.threads.pop(0) @@ -129,13 +128,17 @@ def socket_send(s, func, args = (), kwargs = None): s.sendall(dumper(d)) response = s.recv(buffer_size) if response: - for _ in range(10): # warrants all image/movie is collected + for _ in range(100): # warrants all image/movie is collected try: status, data = loader(response) - except (pickle.UnpicklingError, ValueError, RuntimeError): + except (pickle.UnpicklingError, EOFError, ValueError): response += s.recv(buffer_size) + time.sleep(0.01) else: break + else: + status = 500 + data = (None, None) else: raise RuntimeError('Received empty response when evaluating %s' % d) if status == 200: @@ -155,7 +158,8 @@ def cam_send(self, func, args = (), kwargs = None): def test_20_getHolderType(self): r = self.tem_send('getHolderType') if not isinstance(self.tem_server.device, SimuMicroscope): - self.assertIsInstance(r, self.const.StageHolderType) + self.assertIsInstance(r, int) + self.assertIn(r,list(self.const.StageHolderType.values())) def test_21_getStagePosition(self): r = self.tem_send('getStagePosition') @@ -200,8 +204,15 @@ def test_31_setStagePosition(self): def test_32_setStagePosition(self): self.tem_send('setStagePosition', (10000, 10000, 0, 0, 0)) + self.tem_send('getStagePosition') self.tem_send('setStagePosition', kwargs={'z': 10000}) r = self.tem_send('getStagePosition') + self.assertAlmostEqual(r[2], 10000, delta=PRECISION_NM) + self.tem_send('setStagePosition', kwargs={'z': 0}) + r = self.tem_send('getStagePosition') + self.assertAlmostEqual(r[2], 0, delta=PRECISION_NM) + self.tem_send('setStagePosition', (10000, 10000, 10000, 0, 0)) + r = self.tem_send('getStagePosition') self.assertAlmostEqual(r[0], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[1], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[2], 10000, delta=PRECISION_NM) @@ -215,11 +226,11 @@ def test_33_setStagePosition(self): def test_35_setStagePosition(self): self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'x': 1000}) + self.tem_send('setStagePosition', kwargs={'x': 5000}) t1 = time.perf_counter() self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.1}) t2 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'x': 1000, 'speed': 0.05}) + self.tem_send('setStagePosition', kwargs={'x': 5000, 'speed': 0.05}) t3 = time.perf_counter() self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.02}) t4 = time.perf_counter() @@ -249,7 +260,8 @@ def test_38_setStagePosition(self): t0 = time.perf_counter() self.tem_send('setStagePosition', kwargs={'a': 30, 'wait': False}) t1 = time.perf_counter() - self.tem_send('waitForStage', kwargs={'delay': 0.01}) + time.sleep(0.1) + self.tem_send('waitForStage') t2 = time.perf_counter() q = {'a': 0, 'wait': True} self.tem_send('setStagePosition', kwargs=q) @@ -285,6 +297,7 @@ def test_48_setStageA(self): t0 = time.perf_counter() self.tem_send('setStageA', kwargs={'value': 10, 'wait': False}) t1 = time.perf_counter() + time.sleep(0.1) self.tem_send('waitForStage', kwargs={'delay': 0.01}) t2 = time.perf_counter() self.tem_send('setStageA', kwargs={'value': 0, 'wait': True}) From 3c1cb146201367dd434e304763aee892bef758f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 3 Dec 2025 19:30:29 +0100 Subject: [PATCH 10/26] Rewrite the start.bat to closer match the actual use case --- instamaticServer/start.bat | 3 --- start.bat | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100755 instamaticServer/start.bat create mode 100644 start.bat diff --git a/instamaticServer/start.bat b/instamaticServer/start.bat deleted file mode 100755 index ba3ef5a..0000000 --- a/instamaticServer/start.bat +++ /dev/null @@ -1,3 +0,0 @@ -cd c:\instamatic\TEMserver &:: point to your installation directory -:: call .\venv\Scripts\activate.bat &:: uncomment and point to venv if using one -py tem_server.py &:: use `py`, `py3`, or `python` as needed diff --git a/start.bat b/start.bat new file mode 100644 index 0000000..62a5676 --- /dev/null +++ b/start.bat @@ -0,0 +1,3 @@ +cd c:\instamatic\instamatic-tecnai-server &:: point to your installation directory +:: call .\venv\Scripts\activate.bat &:: uncomment and point to venv if using one +python tem_server.py &:: use `py`, `py3`, or `python` as needed From b8ac7897d2661f3ee0f61012fbeb8fd706a4301d Mon Sep 17 00:00:00 2001 From: erc_user Date: Mon, 5 Jan 2026 16:54:42 +0100 Subject: [PATCH 11/26] E-RC: hotfixes to make things work on Titan --- instamaticServer/TEMController/camera.py | 3 ++- instamaticServer/TEMController/tecnai_camera.py | 12 ++++++++---- instamaticServer/TEMController/tecnai_microscope.py | 4 ++++ instamaticServer/utils/settings.yaml | 12 ++++++------ start.bat | 8 ++++++-- tem_server.py | 5 +++++ tests.py | 5 +++++ 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/instamaticServer/TEMController/camera.py b/instamaticServer/TEMController/camera.py index 95c4761..cdcc239 100644 --- a/instamaticServer/TEMController/camera.py +++ b/instamaticServer/TEMController/camera.py @@ -13,7 +13,8 @@ def get_camera_class(interface: str): if interface == 'simulate': from .simu_camera import SimuCamera as CamCls elif interface == 'tecnai': - from .tecnai_microscope import TecnaiMicroscope as CamCls + from .tecnai_camera import TecnaiCamera as CamCls + # from .tecnai_microscope import TecnaiMicroscope as CamCls else: raise ValueError("No such microscope interface: %s" % interface) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index 735a10e..38fba3d 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import atexit import logging from typing import Tuple, Any, Optional, List @@ -37,7 +35,7 @@ def __init__(self, name='tecnai'): self.name = name self.load_defaults() self.acq, self.cam = self.establish_connection() - logger.info(f'Camera Tecnai connection established') + logger.info('Camera Tecnai connection established') atexit.register(self.release_connection) def __enter__(self): @@ -87,7 +85,13 @@ def get_movie( def establish_connection(self) -> Tuple[Any, Any]: """Establish connection to the camera.""" - acq = TecnaiMicroscope()._tem.Acquisition() + # test code to see if multi-threading is an issue + # import comtypes + # comtypes.CoInitialize() + # tem2 = comtypes.client.CreateObject('TEMScripting.Instrument.1') + # acq = tem2.Acquisition + + # acq = TecnaiMicroscope()._tem.Acquisition # old code acq.RemoveAllAcqDevices() cam = acq.Cameras[0] cam.AcqParams.ImageCorrection = 1 # bias and gain corr (0=off, 1=on) diff --git a/instamaticServer/TEMController/tecnai_microscope.py b/instamaticServer/TEMController/tecnai_microscope.py index 057d0fb..e4ac876 100644 --- a/instamaticServer/TEMController/tecnai_microscope.py +++ b/instamaticServer/TEMController/tecnai_microscope.py @@ -30,6 +30,10 @@ def __init__(self, name: str=None) -> None: logger.info('FEI Scripting initializing...') ## TEM interfaces the GUN, stage etc. + enum constants self._tem = comtypes.client.CreateObject('TEMScripting.Instrument', comtypes.CLSCTX_ALL) + # TEMPORARY TEST LINE + # acq = self._tem.Acquisition + # print(dir(acq)) + # END OF TEST self._tem_constant = comtypes.client.Constants(self._tem) self._t = 0 diff --git a/instamaticServer/utils/settings.yaml b/instamaticServer/utils/settings.yaml index fd8e986..32fc3e8 100644 --- a/instamaticServer/utils/settings.yaml +++ b/instamaticServer/utils/settings.yaml @@ -1,5 +1,5 @@ -microscope: simulate_tem -camera: simulate_cam +microscope: titan +camera: ultrascan calibration: simulate # Global toggle to force simulated camera/microscope interface @@ -11,16 +11,16 @@ flatfield: # Run the TEM connection in a different process (recommended) use_tem_server: True -tem_server_host: 'localhost' +tem_server_host: '192.168.0.1' tem_server_port: 8088 tem_require_admin: False tem_communication_protocol: pickle # pickle, json, msgpack, yaml # Run the Camera connection in a different process -use_cam_server: False -cam_server_host: 'localhost' +use_cam_server: True +cam_server_host: '192.168.0.1' cam_server_port: 8087 -cam_use_shared_memory: true +cam_use_shared_memory: False # Submit collected data to an indexing server (CRED only) use_indexing_server_exe: False diff --git a/start.bat b/start.bat index 62a5676..f03427f 100644 --- a/start.bat +++ b/start.bat @@ -1,3 +1,7 @@ -cd c:\instamatic\instamatic-tecnai-server &:: point to your installation directory +cd Q:\DanielT\instamatic-tecnai-server +Q:\Malika\py\py34\python-3.4.4\python.exe Q:\DanielT\instamatic-tecnai-server\tem_server.py &:: -c + + +:: cd c:\instamatic\instamatic-tecnai-server &:: point to your installation directory :: call .\venv\Scripts\activate.bat &:: uncomment and point to venv if using one -python tem_server.py &:: use `py`, `py3`, or `python` as needed +:: python tem_server.py &:: use `py`, `py3`, or `python` as needed diff --git a/tem_server.py b/tem_server.py index d64a1fa..a2876a7 100644 --- a/tem_server.py +++ b/tem_server.py @@ -1,3 +1,8 @@ +# BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE +import sys +sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') +# BOILERPLATE END + import datetime import logging import queue diff --git a/tests.py b/tests.py index a1ec6cb..3c22d33 100644 --- a/tests.py +++ b/tests.py @@ -1,3 +1,8 @@ +# BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE +import sys +sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') +# BOILERPLATE END + import atexit import pickle import socket From 182d838bc9c850e95fcea44d22d5a61e1e5a02ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 09:26:39 +0100 Subject: [PATCH 12/26] E-RC: split joint tem server into tem and cam servers working in separate processes --- cam_server.py | 107 ++++++++++++++++++ instamaticServer/TEMController/camera.py | 1 - .../TEMController/tecnai_camera.py | 13 +-- .../TEMController/tecnai_microscope.py | 13 +-- tem_server.py | 49 +------- 5 files changed, 120 insertions(+), 63 deletions(-) create mode 100644 cam_server.py diff --git a/cam_server.py b/cam_server.py new file mode 100644 index 0000000..3c317dc --- /dev/null +++ b/cam_server.py @@ -0,0 +1,107 @@ +# BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE +import sys +sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') +# BOILERPLATE END + +import argparse +import datetime +import logging +import queue +import sys +import threading +import time + +from instamaticServer.TEMController.camera import get_camera +from tem_server import DeviceServer, _conf, listen + + +stop_program_event = threading.Event() +BUFSIZE = 1024 +TIMEOUT = 0.5 + + +class MicrosecondFormatter(logging.Formatter): + def formatTime(self, record, datefmt=None): + t = datetime.datetime.fromtimestamp(record.created) + return t.strftime(datefmt) if datefmt else t.strftime("%H:%M:%S.%f") + + +logfile = 'cam_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') +logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' +logging.basicConfig(level=15, filename=logfile, format=logging_fmt) +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.setFormatter(MicrosecondFormatter(logging_fmt)) +logging.getLogger().addHandler(stdout_handler) +logging.addLevelName(15, "EVAL") + + +class CamServer(DeviceServer): + """FEI Tecnai/Titan Acquisition camera communication server.""" + + device_abbr = 'cam' + device_kind = 'camera' + device_getter = staticmethod(get_camera) + requests = queue.Queue(maxsize=1) + responses = queue.Queue(maxsize=1) + host = _conf.default_settings['cam_server_host'] + port = _conf.default_settings['cam_server_port'] + + def __init__(self, name=None) -> None: + super(CamServer, self).__init__(name=name) + self.logger.setLevel(logging.INFO) + + +def main() -> None: + """ + Connects to the TEM and starts a server for camera communication. + Opens a socket on port {CamServer.host}:{CamServer.port}. + + This program initializes a connection to the TEM as defined in the config. + The purpose of this program is to isolate the camera connection in + a separate process for improved stability of the interface in case + instamatic crashes or is started and stopped frequently. + For running the GUI, the cam server is required. + Another reason is that it allows for remote connections from different PCs. + The connection goes over a TCP socket. + + The host and port are defined in `config/settings.yaml`. + + The data sent over the socket is a serialized dictionary with the following: + + - `attr_name`: Name of the function to call (str) + - `args`: (Optional) List of arguments for the function (list) + - `kwargs`: (Optiona) Dictionary of keyword arguments for the function (dict) + + The response is returned as a serialized object. + """ + + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument('-c', '--camera', action='store', + help='Override camera to use.') + parser.set_defaults(camera=None) + options = parser.parse_args() + + logging.info('Titan camera server starting') + + cam_server = CamServer(name=options.camera) + cam_server.start() + + cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') + cam_listener.start() + + threads = [cam_server, cam_listener] + + try: + while not stop_program_event.is_set(): time.sleep(TIMEOUT) + except KeyboardInterrupt: + logging.info("Received KeyboardInterrupt, shutting down...") + finally: + stop_program_event.set() + for thread in threads: + thread.join() + logging.info('Titan camera server terminating') + logging.shutdown() + + +if __name__ == '__main__': + main() diff --git a/instamaticServer/TEMController/camera.py b/instamaticServer/TEMController/camera.py index cdcc239..ff296cb 100644 --- a/instamaticServer/TEMController/camera.py +++ b/instamaticServer/TEMController/camera.py @@ -14,7 +14,6 @@ def get_camera_class(interface: str): from .simu_camera import SimuCamera as CamCls elif interface == 'tecnai': from .tecnai_camera import TecnaiCamera as CamCls - # from .tecnai_microscope import TecnaiMicroscope as CamCls else: raise ValueError("No such microscope interface: %s" % interface) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index 38fba3d..745f49c 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -1,8 +1,8 @@ import atexit +import comtypes.client import logging from typing import Tuple, Any, Optional, List -from instamaticServer.TEMController.tecnai_microscope import TecnaiMicroscope from instamaticServer.utils.config import config from instamaticServer.utils.singleton import Singleton @@ -32,6 +32,9 @@ class TecnaiCamera(metaclass=Singleton): def __init__(self, name='tecnai'): """Initialize camera module.""" + comtypes.CoInitialize() + logger.info('FEI Scripting initializing...') + self._tem = comtypes.client.CreateObject('TEMScripting.Instrument', comtypes.CLSCTX_ALL) self.name = name self.load_defaults() self.acq, self.cam = self.establish_connection() @@ -85,13 +88,7 @@ def get_movie( def establish_connection(self) -> Tuple[Any, Any]: """Establish connection to the camera.""" - # test code to see if multi-threading is an issue - # import comtypes - # comtypes.CoInitialize() - # tem2 = comtypes.client.CreateObject('TEMScripting.Instrument.1') - # acq = tem2.Acquisition - - # acq = TecnaiMicroscope()._tem.Acquisition # old code + acq = self._tem.Acquisition acq.RemoveAllAcqDevices() cam = acq.Cameras[0] cam.AcqParams.ImageCorrection = 1 # bias and gain corr (0=off, 1=on) diff --git a/instamaticServer/TEMController/tecnai_microscope.py b/instamaticServer/TEMController/tecnai_microscope.py index e4ac876..c130dca 100644 --- a/instamaticServer/TEMController/tecnai_microscope.py +++ b/instamaticServer/TEMController/tecnai_microscope.py @@ -1,7 +1,7 @@ import atexit +import comtypes.client import logging import time -import comtypes.client from math import pi from typing import Optional @@ -21,19 +21,10 @@ class TecnaiMicroscope(metaclass=Singleton): """Python bindings to the Tecnai-G2 microscope using the COM scripting interface.""" def __init__(self, name: str=None) -> None: - - try: - comtypes.CoInitialize() - except: - raise - + comtypes.CoInitialize() logger.info('FEI Scripting initializing...') ## TEM interfaces the GUN, stage etc. + enum constants self._tem = comtypes.client.CreateObject('TEMScripting.Instrument', comtypes.CLSCTX_ALL) - # TEMPORARY TEST LINE - # acq = self._tem.Acquisition - # print(dir(acq)) - # END OF TEST self._tem_constant = comtypes.client.Constants(self._tem) self._t = 0 diff --git a/tem_server.py b/tem_server.py index a2876a7..71a64bc 100644 --- a/tem_server.py +++ b/tem_server.py @@ -3,6 +3,7 @@ sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') # BOILERPLATE END +import argparse import datetime import logging import queue @@ -14,7 +15,7 @@ from typing import Any, Type, Callable -from instamaticServer.TEMController.camera import get_camera + from instamaticServer.TEMController.microscope import get_microscope from instamaticServer.serializer import dumper, loader from instamaticServer.utils.config import config @@ -35,7 +36,7 @@ def formatTime(self, record, datefmt=None): logfile = 'tem_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' -logging.basicConfig(level=15, filename='tem_server.log', format=logging_fmt) +logging.basicConfig(level=15, filename=logfile, format=logging_fmt) stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(MicrosecondFormatter(logging_fmt)) logging.getLogger().addHandler(stdout_handler) @@ -136,22 +137,6 @@ class TemServer(DeviceServer): port = _conf.default_settings['tem_server_port'] -class CamServer(DeviceServer): - """FEI Tecnai/Titan Acquisition camera communication server.""" - - device_abbr = 'cam' - device_kind = 'camera' - device_getter = staticmethod(get_camera) - requests = queue.Queue(maxsize=1) - responses = queue.Queue(maxsize=1) - host = _conf.default_settings['cam_server_host'] - port = _conf.default_settings['cam_server_port'] - - def __init__(self, name=None) -> None: - super(CamServer, self).__init__(name=name) - self.logger.setLevel(logging.INFO) - - def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: """Handle incoming connection, put command on the Queue `q`, which is then handled by TEMServer.""" @@ -205,7 +190,7 @@ def listen(server_type: Type[DeviceServer]) -> None: def main() -> None: """ Connects to the TEM and starts a server for microscope communication. - Opens a socket on port {HOST}:{PORT}. + Opens a socket on port {TemServer.host}:{TemServer.port}. This program initializes a connection to the TEM as defined in the config. The purpose of this program is to isolate the microscope connection in @@ -225,19 +210,14 @@ def main() -> None: The response is returned as a serialized object. """ - - import argparse parser = argparse.ArgumentParser(description=main.__doc__) parser.add_argument('-t', '--microscope', action='store', help='Override microscope to use.') - parser.add_argument('-c', '--camera', action='store_true', - help='If selected, start separate threads for a camera') - parser.set_defaults(microscope=None) options = parser.parse_args() - logging.info('Tecnai server starting') + logging.info('Tecnai microscope server starting') tem_server = TemServer(name=options.microscope) tem_server.start() @@ -247,23 +227,6 @@ def main() -> None: threads = [tem_server, tem_listener] - if options.camera: - logging.info('Waiting for the TEM singleton to initialize') - for _ in range(100): - if getattr(tem_server, 'device') is not None: # wait until TEM initialized - break - time.sleep(0.05) - else: # necessary check, Error extremely unlikely, TEM typically starts in ms - raise RuntimeError('Could not start TEM device on server in 5 seconds') - - cam_server = CamServer(name=options.microscope) - cam_server.start() - - cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') - cam_listener.start() - - threads.extend([cam_server, cam_listener]) - try: while not stop_program_event.is_set(): time.sleep(TIMEOUT) except KeyboardInterrupt: @@ -272,7 +235,7 @@ def main() -> None: stop_program_event.set() for thread in threads: thread.join() - logging.info('Tecnai server terminating') + logging.info('Tecnai microscope server terminating') logging.shutdown() From 7c6d7d96caaf9ad8011739ab8c8c924b10169d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 10:14:38 +0100 Subject: [PATCH 13/26] E-RC: address logging, stop event cross-contamination between two new programs --- cam_server.py | 37 ++++++++------------------- tem_server.py | 70 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 56 deletions(-) diff --git a/cam_server.py b/cam_server.py index 3c317dc..b64f72e 100644 --- a/cam_server.py +++ b/cam_server.py @@ -4,7 +4,6 @@ # BOILERPLATE END import argparse -import datetime import logging import queue import sys @@ -12,29 +11,13 @@ import time from instamaticServer.TEMController.camera import get_camera -from tem_server import DeviceServer, _conf, listen +from tem_server import DeviceServer, _conf, listen, setup_logging -stop_program_event = threading.Event() BUFSIZE = 1024 TIMEOUT = 0.5 -class MicrosecondFormatter(logging.Formatter): - def formatTime(self, record, datefmt=None): - t = datetime.datetime.fromtimestamp(record.created) - return t.strftime(datefmt) if datefmt else t.strftime("%H:%M:%S.%f") - - -logfile = 'cam_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') -logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' -logging.basicConfig(level=15, filename=logfile, format=logging_fmt) -stdout_handler = logging.StreamHandler(sys.stdout) -stdout_handler.setFormatter(MicrosecondFormatter(logging_fmt)) -logging.getLogger().addHandler(stdout_handler) -logging.addLevelName(15, "EVAL") - - class CamServer(DeviceServer): """FEI Tecnai/Titan Acquisition camera communication server.""" @@ -43,6 +26,7 @@ class CamServer(DeviceServer): device_getter = staticmethod(get_camera) requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) + stop_event = threading.Event() host = _conf.default_settings['cam_server_host'] port = _conf.default_settings['cam_server_port'] @@ -81,7 +65,8 @@ def main() -> None: parser.set_defaults(camera=None) options = parser.parse_args() - logging.info('Titan camera server starting') + logger = setup_logging(device_abbr='cam') + logger.info('Titan camera server starting') cam_server = CamServer(name=options.camera) cam_server.start() @@ -89,17 +74,15 @@ def main() -> None: cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') cam_listener.start() - threads = [cam_server, cam_listener] - try: - while not stop_program_event.is_set(): time.sleep(TIMEOUT) + while not CamServer.stop_event.is_set(): time.sleep(TIMEOUT) except KeyboardInterrupt: - logging.info("Received KeyboardInterrupt, shutting down...") + logger.info("Received KeyboardInterrupt, shutting down...") finally: - stop_program_event.set() - for thread in threads: - thread.join() - logging.info('Titan camera server terminating') + CamServer.stop_event.set() + cam_server.join() + cam_listener.join() + logger.info('Titan camera server terminating') logging.shutdown() diff --git a/tem_server.py b/tem_server.py index 71a64bc..b28d44f 100644 --- a/tem_server.py +++ b/tem_server.py @@ -21,27 +21,13 @@ from instamaticServer.utils.config import config -stop_program_event = threading.Event() +logging.addLevelName(15, "EVAL") _conf = config() BUFSIZE = 1024 TIMEOUT = 0.5 -class MicrosecondFormatter(logging.Formatter): - def formatTime(self, record, datefmt=None): - t = datetime.datetime.fromtimestamp(record.created) - return t.strftime(datefmt) if datefmt else t.strftime("%H:%M:%S.%f") - - -logfile = 'tem_server_%s.log' % datetime.datetime.now().strftime('%Y-%m-%d') -logging_fmt = '%(asctime)s %(name)-4s: %(levelname)-8s %(message)s' -logging.basicConfig(level=15, filename=logfile, format=logging_fmt) -stdout_handler = logging.StreamHandler(sys.stdout) -stdout_handler.setFormatter(MicrosecondFormatter(logging_fmt)) -logging.getLogger().addHandler(stdout_handler) -logging.addLevelName(15, "EVAL") - def log_eval(logger, status, func_name, args, kwargs, ret): if logger.isEnabledFor(15): args_list = [repr(a) for a in args] @@ -64,13 +50,14 @@ class DeviceServer(threading.Thread): device_getter = None # type: Callable requests = None # type: queue.Queue responses = None # type: queue.Queue + stop_event = None # type: threading.Event host = 'localhost' # type: str port = None # type: int def __init__(self, name = None) -> None: super().__init__(name=self.device_kind + '_server') self.interface_name = name - self.logger = logging.getLogger(self.device_abbr + 'S') # temS/camS server + self.logger = logging.getLogger(self.device_abbr) # temS/camS server self.device = None self.verbose = False @@ -84,7 +71,7 @@ def run(self) -> None: try: cmd = self.requests.get(timeout=TIMEOUT) except queue.Empty: - if stop_program_event.is_set(): + if self.stop_event.is_set(): break continue @@ -133,6 +120,7 @@ class TemServer(DeviceServer): device_getter = staticmethod(get_microscope) requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) + stop_event = threading.Event() host = _conf.default_settings['tem_server_host'] port = _conf.default_settings['tem_server_port'] @@ -143,7 +131,7 @@ def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: with conn: conn.settimeout(TIMEOUT) while True: - if stop_program_event.is_set(): + if server_type.stop_event.is_set(): break try: @@ -168,14 +156,14 @@ def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: def listen(server_type: Type[DeviceServer]) -> None: """Listen on a given server host/port and handle incoming instructions""" - logger = logging.getLogger(server_type.device_abbr + 'L') # temL/camL listener + logger = logging.getLogger(server_type.device_abbr) # tem/cam listener with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as device_client: device_client.bind((server_type.host, server_type.port)) device_client.settimeout(TIMEOUT) device_client.listen(1) logger.info('Server listening on %s:%s', server_type.host, server_type.port) while True: - if stop_program_event.is_set(): + if server_type.stop_event.is_set(): break try: connection, _ = device_client.accept() @@ -187,6 +175,31 @@ def listen(server_type: Type[DeviceServer]) -> None: logger.info('Terminating %s listener thread', server_type.device_kind) +class MicrosecondFormatter(logging.Formatter): + def formatTime(self, record, datefmt=None): + t = datetime.datetime.fromtimestamp(record.created) + return t.strftime(datefmt) if datefmt else t.strftime("%H:%M:%S.%f") + + +def setup_logging(device_abbr='tem') -> logging.Logger: + logger = logging.getLogger(device_abbr) + logger.setLevel(15) + + logfile = '%s_server_%s.log' % (device_abbr, datetime.datetime.now().strftime('%Y-%m-%d')) + fh = logging.FileHandler(logfile) + sh = logging.StreamHandler(sys.stdout) + + fmt = MicrosecondFormatter('%(asctime)s %(name)-3s: %(levelname)-8s %(message)s') + fh.setFormatter(fmt) + sh.setFormatter(fmt) + + logger.addHandler(fh) + logger.addHandler(sh) + + logger.propagate = False + return logger + + def main() -> None: """ Connects to the TEM and starts a server for microscope communication. @@ -217,7 +230,8 @@ def main() -> None: parser.set_defaults(microscope=None) options = parser.parse_args() - logging.info('Tecnai microscope server starting') + logger = setup_logging(device_abbr='tem') + logger.info('Tecnai microscope server starting') tem_server = TemServer(name=options.microscope) tem_server.start() @@ -225,17 +239,15 @@ def main() -> None: tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') tem_listener.start() - threads = [tem_server, tem_listener] - try: - while not stop_program_event.is_set(): time.sleep(TIMEOUT) + while not TemServer.stop_event.is_set(): time.sleep(TIMEOUT) except KeyboardInterrupt: - logging.info("Received KeyboardInterrupt, shutting down...") + logger.info("Received KeyboardInterrupt, shutting down...") finally: - stop_program_event.set() - for thread in threads: - thread.join() - logging.info('Tecnai microscope server terminating') + TemServer.stop_event.set() + tem_server.join() + tem_listener.join() + logger.info('Tecnai microscope server terminating') logging.shutdown() From 085aa780e6228ec744bd8dd6f72537dbdecea3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 11:41:55 +0100 Subject: [PATCH 14/26] E-RC: get_image: respect default binsize, fix get_image_dimensions typo --- instamaticServer/TEMController/tecnai_camera.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index 745f49c..34afcef 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -62,10 +62,10 @@ def load_defaults(self) -> None: for key, val in _conf.camera.__dict__.items(): setattr(self, key, val) - def get_image(self, exposure: Optional[float] = None, binning: int = 1): + def get_image(self, exposure: Optional[float] = None, binning: Optional[int] = None): """Image acquisition interface.""" self.cam.AcqParams.ExposureTime = exposure or self.default_exposure - self.cam.AcqParams.Binning = binning + self.cam.AcqParams.Binning = binning or self.default_binsize img = self.acq.AcquireImages()[0] sa = img.AsSafeArray if np: @@ -75,7 +75,8 @@ def get_image(self, exposure: Optional[float] = None, binning: int = 1): def get_image_dimensions(self) -> Tuple[int, int]: """Get the binned dimensions reported by the camera.""" - return self.cam.ImageSize + dx, dy = self.cam.AcqParams.ImageSize[:2] + return dx // self.default_binsize, dy // self.default_binsize def get_movie( self, From fee74036b35824f58cecd6ee6953b3f086498c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 11:49:35 +0100 Subject: [PATCH 15/26] E-RC: get_image: fix get_image_dimensions bug (determine locally) --- instamaticServer/TEMController/tecnai_camera.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index 34afcef..cd079c3 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -74,8 +74,8 @@ def get_image(self, exposure: Optional[float] = None, binning: Optional[int] = N # try [[sa.GetElement([r, c]) or similar if direct indexing does not work... def get_image_dimensions(self) -> Tuple[int, int]: - """Get the binned dimensions reported by the camera.""" - dx, dy = self.cam.AcqParams.ImageSize[:2] + """Get the binned dimensions of the camera.""" + dx, dy = self.dimensions return dx // self.default_binsize, dy // self.default_binsize def get_movie( @@ -92,7 +92,7 @@ def establish_connection(self) -> Tuple[Any, Any]: acq = self._tem.Acquisition acq.RemoveAllAcqDevices() cam = acq.Cameras[0] - cam.AcqParams.ImageCorrection = 1 # bias and gain corr (0=off, 1=on) + cam.AcqParams.ImageCorrection = 0 # bias and gain corr (0=off, 1=on) cam.AcqParams.ImageSize = 0 # sub area centered (0=full, 1=half, 2=quarter) acq.AddAcqDeviceByName(cam.Info.Name) return acq, cam From f1a1d7ed083fcdefc44d119479fe7270c871eff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 14:20:22 +0100 Subject: [PATCH 16/26] E-RC: non-callable remote camera attrs must be private if non-picklable --- instamaticServer/TEMController/tecnai_camera.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index cd079c3..a3a5328 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -37,7 +37,7 @@ def __init__(self, name='tecnai'): self._tem = comtypes.client.CreateObject('TEMScripting.Instrument', comtypes.CLSCTX_ALL) self.name = name self.load_defaults() - self.acq, self.cam = self.establish_connection() + self._acq, self._cam = self.establish_connection() logger.info('Camera Tecnai connection established') atexit.register(self.release_connection) @@ -64,9 +64,9 @@ def load_defaults(self) -> None: def get_image(self, exposure: Optional[float] = None, binning: Optional[int] = None): """Image acquisition interface.""" - self.cam.AcqParams.ExposureTime = exposure or self.default_exposure - self.cam.AcqParams.Binning = binning or self.default_binsize - img = self.acq.AcquireImages()[0] + self._cam.AcqParams.ExposureTime = exposure or self.default_exposure + self._cam.AcqParams.Binning = binning or self.default_binsize + img = self._acq.AcquireImages()[0] sa = img.AsSafeArray if np: return np.stack(sa).T @@ -92,7 +92,7 @@ def establish_connection(self) -> Tuple[Any, Any]: acq = self._tem.Acquisition acq.RemoveAllAcqDevices() cam = acq.Cameras[0] - cam.AcqParams.ImageCorrection = 0 # bias and gain corr (0=off, 1=on) + cam.AcqParams.ImageCorrection = 1 # bias and gain corr (0=off, 1=on) cam.AcqParams.ImageSize = 0 # sub area centered (0=full, 1=half, 2=quarter) acq.AddAcqDeviceByName(cam.Info.Name) return acq, cam From 6954646cb79c3ce23ee373f8e26844694d6a4e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 14:41:51 +0100 Subject: [PATCH 17/26] E-RC: any subclasses of simple namespace are not pickle-able and must be re-created locally --- instamaticServer/utils/config.py | 11 +++-------- tests.py | 10 +++++----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/instamaticServer/utils/config.py b/instamaticServer/utils/config.py index 29b98e4..9da0297 100644 --- a/instamaticServer/utils/config.py +++ b/instamaticServer/utils/config.py @@ -8,15 +8,10 @@ _settings_file = 'settings.yaml' -class NS(SimpleNamespace): - def get(self, key, default=None): - return getattr(self, key, default) - - -def dict_to_namespace(d: Dict) -> NS: +def dict_to_namespace(d: Dict) -> SimpleNamespace: """Recursively converts a dictionary into a SimpleNamespace.""" if isinstance(d, dict): - return NS(**{k: dict_to_namespace(v) for k, v in d.items()}) + return SimpleNamespace(**{k: dict_to_namespace(v) for k, v in d.items()}) return d @@ -32,7 +27,7 @@ def __init__(self, name:str=None): try: self.camera = self.load_camera_config() except FileNotFoundError: - self.camera = NS() + self.camera = SimpleNamespace() def settings(self) -> dict: """load the settings.yaml file.""" diff --git a/tests.py b/tests.py index 3c22d33..0531595 100644 --- a/tests.py +++ b/tests.py @@ -11,8 +11,8 @@ import unittest from instamaticServer.TEMController.simu_microscope import SimuMicroscope -from instamaticServer.utils.config import NS, config, dict_to_namespace -from tem_server import stop_program_event +from instamaticServer.utils.config import SimpleNamespace, config, dict_to_namespace +from tem_server import stop_program_event # TODO: this is outdated _conf_dict = {'a': 1, 'b': {'c': 3, 'd': 4}} _conf = config() @@ -23,19 +23,19 @@ class TestConfig(unittest.TestCase): def test_namespace(self): - ns = NS(**_conf_dict) + ns = SimpleNamespace(**_conf_dict) self.assertEqual(ns.a, 1) self.assertEqual(ns.b, {'c': 3, 'd': 4}) def test_dict_to_namespace(self): ns = dict_to_namespace(_conf_dict) self.assertEqual(ns.a, 1) - self.assertEqual(ns.b, NS(**{'c': 3, 'd': 4})) + self.assertEqual(ns.b, SimpleNamespace(**{'c': 3, 'd': 4})) def test_config(self): global _conf self.assertIn(_conf.micr_interface, {'tecnai', 'simulate'}) - self.assertIsInstance(_conf.camera, NS) + self.assertIsInstance(_conf.camera, SimpleNamespace) class TestSerializers(unittest.TestCase): From ed3e6d3496ad71cc8c8696dda47951b42de94d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 14:43:58 +0100 Subject: [PATCH 18/26] E-RC: any subclasses of simple namespace are not pickle-able and must be re-created locally 2 --- instamaticServer/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instamaticServer/utils/config.py b/instamaticServer/utils/config.py index 9da0297..8a7cd82 100644 --- a/instamaticServer/utils/config.py +++ b/instamaticServer/utils/config.py @@ -49,7 +49,7 @@ def microscope(self): return interface, wavelength, micr_ranges - def load_camera_config(self) -> NS: + def load_camera_config(self) -> SimpleNamespace: directory = Path(__file__).resolve().parent file = directory / (str(self.default_settings['camera']) + '.yaml') with open(str(file), 'r') as stream: From 5dbdbe3d8cc725b234668191213c35899bfd2b11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 14:51:36 +0100 Subject: [PATCH 19/26] E-RC: fix get_image typo (binning -> binsize) --- instamaticServer/TEMController/tecnai_camera.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index a3a5328..9eb0263 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -62,10 +62,10 @@ def load_defaults(self) -> None: for key, val in _conf.camera.__dict__.items(): setattr(self, key, val) - def get_image(self, exposure: Optional[float] = None, binning: Optional[int] = None): + def get_image(self, exposure: Optional[float] = None, binsize: Optional[int] = None): """Image acquisition interface.""" self._cam.AcqParams.ExposureTime = exposure or self.default_exposure - self._cam.AcqParams.Binning = binning or self.default_binsize + self._cam.AcqParams.Binning = binsize or self.default_binsize img = self._acq.AcquireImages()[0] sa = img.AsSafeArray if np: From 7a5ebc9ab5fca01787a2cb26dc72c476d41016cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Thu, 8 Jan 2026 11:55:58 +0100 Subject: [PATCH 20/26] E-RC: patch instamatic RPC to allow get_movie generator --- .../TEMController/tecnai_camera.py | 12 +++++++---- tem_server.py | 21 ++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index 9eb0263..3510faf 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -1,8 +1,11 @@ import atexit +from functools import partial + import comtypes.client import logging -from typing import Tuple, Any, Optional, List +from typing import Any, Generator, List, Optional, Tuple +from instamaticServer.TEMController.movie_thread import RemoteMovie from instamaticServer.utils.config import config from instamaticServer.utils.singleton import Singleton @@ -83,9 +86,10 @@ def get_movie( n_frames: int, exposure: Optional[float] = None, binsize: Optional[int] = None, - ): - """Unfortunately not designed to work with generators, as a server...""" - return [self.get_image(exposure, binsize) for _ in range(n_frames)] + ) -> Generator: + """Yield n_frames images each collected using subsequent get_image""" + get_image = partial(self.get_image, self) + yield from RemoteMovie(get_image, n_frames=n_frames, exposure=exposure, binsize=binsize) def establish_connection(self) -> Tuple[Any, Any]: """Establish connection to the camera.""" diff --git a/tem_server.py b/tem_server.py index b28d44f..7d37b1c 100644 --- a/tem_server.py +++ b/tem_server.py @@ -5,6 +5,7 @@ import argparse import datetime +import inspect import logging import queue import socket @@ -12,6 +13,7 @@ import threading import time import traceback +import uuid from typing import Any, Type, Callable @@ -22,8 +24,8 @@ logging.addLevelName(15, "EVAL") - _conf = config() +_generators = {} BUFSIZE = 1024 TIMEOUT = 0.5 @@ -82,6 +84,10 @@ def run(self) -> None: try: ret = self.evaluate(func_name, args, kwargs) status = 200 + if inspect.isgenerator(ret): + gen_id = uuid.uuid4().hex + _generators[gen_id] = ret + ret = {'__generator__': gen_id} except Exception as e: traceback.print_exc() self.logger.exception(e) @@ -96,6 +102,19 @@ def run(self) -> None: def evaluate(self, func_name: str, args: list, kwargs: dict) -> Any: """Eval function `func_name` on `self.device` with `args` & `kwargs`.""" self.logger.debug('evaluate(func_name=%s, args=%s, kwargs=%s)', func_name, args, kwargs) + + if func_name == '__gen_next__': + gen = _generators[kwargs['id']] + try: + return next(gen) + except StopIteration: + del _generators[kwargs['id']] + return + + if func_name == "__gen_close__": + _generators.pop(kwargs['id'], None) + return + f = getattr(self.device, func_name) return f(*args, **kwargs) if callable(f) else f From 6405f65b3f4777b9775756f87d9193d53e3ada9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Thu, 8 Jan 2026 13:10:31 +0100 Subject: [PATCH 21/26] E-RC: patch instamatic RPC to allow get_movie generator (add file) --- .../TEMController/movie_thread.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 instamaticServer/TEMController/movie_thread.py diff --git a/instamaticServer/TEMController/movie_thread.py b/instamaticServer/TEMController/movie_thread.py new file mode 100644 index 0000000..d91aa35 --- /dev/null +++ b/instamaticServer/TEMController/movie_thread.py @@ -0,0 +1,41 @@ +import queue +import threading +from typing import Any, Callable, Dict + + +class MovieThread(threading.Thread): + """Collects movie images and keeps them in a thread-safe queue.""" + + def __init__(self, get_image: Callable, **kwargs) -> None: + super().__init__() + self.daemon = True + self._get_image = get_image + self._kwargs: Dict[str, Any] = dict(kwargs) + self._n_frames = self._kwargs.pop('n_frames') + self.exception = None + self.queue = queue.Queue() + + def run(self): + try: + for _ in range(self._n_frames): + self.queue.put(self._get_image(**self._kwargs)) + except Exception as e: + self.exception = e + finally: + self.queue.put(None) + + +class RemoteMovie: + """A wrapper that makes the movie thread into a generator""" + def __init__(self, get_image: Callable, **kwargs) -> None: + self.thread = MovieThread(get_image, **kwargs) + + def next(self): + if not self.thread.is_alive(): + self.thread.start() + image = self.thread.queue.get() + if image is None: + if self.thread.exception: + raise self.thread.exception + raise StopIteration + return image From 3bba7cfb41bdff5234bbfb65d01ed56058c53d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Thu, 8 Jan 2026 13:36:10 +0100 Subject: [PATCH 22/26] E-RC: patch instamatic RPC to allow get_movie, but only yield lazily due to multithreading... --- .../TEMController/movie_thread.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/instamaticServer/TEMController/movie_thread.py b/instamaticServer/TEMController/movie_thread.py index d91aa35..ff0f683 100644 --- a/instamaticServer/TEMController/movie_thread.py +++ b/instamaticServer/TEMController/movie_thread.py @@ -26,16 +26,20 @@ def run(self): class RemoteMovie: - """A wrapper that makes the movie thread into a generator""" + """A lazy single-threaded image iterator, can't pass acq to other thread.""" def __init__(self, get_image: Callable, **kwargs) -> None: - self.thread = MovieThread(get_image, **kwargs) - - def next(self): - if not self.thread.is_alive(): - self.thread.start() - image = self.thread.queue.get() - if image is None: - if self.thread.exception: - raise self.thread.exception + self._get_image = get_image + self._kwargs: Dict[str, Any] = dict(kwargs) + self._n_frames = self._kwargs.pop('n_frames') + self._i = 0 + self._started = False + + def __iter__(self): + return self + + def __next__(self): + if self._i >= self._n_frames: raise StopIteration - return image + self._started = True + self._i += 1 + return self._get_image(**self._kwargs) From 5db5fb4ce61e2394c8b11f8866981b57ef4415b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Thu, 8 Jan 2026 13:40:36 +0100 Subject: [PATCH 23/26] E-RC: patch instamatic RPC to allow get_movie, but only yield lazily due to multithreading... 2 --- .../TEMController/movie_thread.py | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/instamaticServer/TEMController/movie_thread.py b/instamaticServer/TEMController/movie_thread.py index ff0f683..51d354f 100644 --- a/instamaticServer/TEMController/movie_thread.py +++ b/instamaticServer/TEMController/movie_thread.py @@ -1,35 +1,11 @@ -import queue -import threading from typing import Any, Callable, Dict -class MovieThread(threading.Thread): - """Collects movie images and keeps them in a thread-safe queue.""" - - def __init__(self, get_image: Callable, **kwargs) -> None: - super().__init__() - self.daemon = True - self._get_image = get_image - self._kwargs: Dict[str, Any] = dict(kwargs) - self._n_frames = self._kwargs.pop('n_frames') - self.exception = None - self.queue = queue.Queue() - - def run(self): - try: - for _ in range(self._n_frames): - self.queue.put(self._get_image(**self._kwargs)) - except Exception as e: - self.exception = e - finally: - self.queue.put(None) - - class RemoteMovie: """A lazy single-threaded image iterator, can't pass acq to other thread.""" def __init__(self, get_image: Callable, **kwargs) -> None: self._get_image = get_image - self._kwargs: Dict[str, Any] = dict(kwargs) + self._kwargs = dict(kwargs) self._n_frames = self._kwargs.pop('n_frames') self._i = 0 self._started = False From ddd8ab36d79a6126b92e6723c0b4ddcfe078f6fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Mon, 12 Jan 2026 20:54:04 +0100 Subject: [PATCH 24/26] ER-C: adapting patches for production --- cam_server.py | 4 ++-- .../TEMController/{movie_thread.py => remote_movie.py} | 0 instamaticServer/TEMController/tecnai_camera.py | 2 +- tem_server.py | 4 ++-- tests.py | 5 ++--- 5 files changed, 7 insertions(+), 8 deletions(-) rename instamaticServer/TEMController/{movie_thread.py => remote_movie.py} (100%) diff --git a/cam_server.py b/cam_server.py index b64f72e..0779bea 100644 --- a/cam_server.py +++ b/cam_server.py @@ -1,6 +1,6 @@ # BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE -import sys -sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') +# import sys +# sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') # BOILERPLATE END import argparse diff --git a/instamaticServer/TEMController/movie_thread.py b/instamaticServer/TEMController/remote_movie.py similarity index 100% rename from instamaticServer/TEMController/movie_thread.py rename to instamaticServer/TEMController/remote_movie.py diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index 3510faf..b8076d0 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -5,7 +5,7 @@ import logging from typing import Any, Generator, List, Optional, Tuple -from instamaticServer.TEMController.movie_thread import RemoteMovie +from instamaticServer.TEMController.remote_movie import RemoteMovie from instamaticServer.utils.config import config from instamaticServer.utils.singleton import Singleton diff --git a/tem_server.py b/tem_server.py index 7d37b1c..7e20740 100644 --- a/tem_server.py +++ b/tem_server.py @@ -1,6 +1,6 @@ # BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE -import sys -sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') +# import sys +# sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') # BOILERPLATE END import argparse diff --git a/tests.py b/tests.py index 0531595..a271e32 100644 --- a/tests.py +++ b/tests.py @@ -1,6 +1,6 @@ # BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE -import sys -sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') +# import sys +# sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') # BOILERPLATE END import atexit @@ -12,7 +12,6 @@ from instamaticServer.TEMController.simu_microscope import SimuMicroscope from instamaticServer.utils.config import SimpleNamespace, config, dict_to_namespace -from tem_server import stop_program_event # TODO: this is outdated _conf_dict = {'a': 1, 'b': {'c': 3, 'd': 4}} _conf = config() From 0ca1e00251b2aea89fc3ef5b2bfddb17d37fbffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 14 Jan 2026 16:49:24 +0100 Subject: [PATCH 25/26] Run Config ~once, update README, simplify settings, fix tests --- README.md | 25 +- cam_server.py | 10 +- instamaticServer/TEMController/camera.py | 5 +- instamaticServer/TEMController/microscope.py | 5 +- instamaticServer/TEMController/simu_camera.py | 3 +- .../TEMController/simu_microscope.py | 8 +- .../TEMController/tecnai_camera.py | 3 +- .../TEMController/tecnai_microscope.py | 11 +- instamaticServer/serializer.py | 5 +- instamaticServer/utils/config.py | 8 +- instamaticServer/utils/exceptions.py | 2 +- instamaticServer/utils/settings.yaml | 61 +--- instamaticServer/utils/titan.yaml | 9 + instamaticServer/utils/ultrascan.yaml | 21 ++ requirements.txt | 1 + start.bat | 13 +- tem_server.py | 7 +- tests.py | 311 +++++++++--------- 18 files changed, 239 insertions(+), 269 deletions(-) create mode 100644 instamaticServer/utils/titan.yaml create mode 100644 instamaticServer/utils/ultrascan.yaml diff --git a/README.md b/README.md index 0397ff9..8a68108 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,32 @@ # Server for the Tecnai-Scripting interface -The "Tecnai-Server" is an additional TEM interface used to control a FEI Tecnai-TEM via the instamatic software. We have tested the server software on a FEI Tecnai G2 microscope. The program provides access for the instamatic software to the com-scripting interface of the TEM. +This "Tecnai-Server" is a supplementary TEM server used to control an FEI machine: Tecnai TEM, Titan TEM, or an integrated camera, via the instamatic software. It can be used in lieu of the full instamatic on computers that do not support modern Python. It has been tested on an FEI Tecnai G2 and an FEI Titan 80-300 microscope. The program provides access for the instamatic software to the com-scripting interface of the TEM. ## Installation and Requirement The server software was developed in a Python 3.4 software environment (Windows XP). Following you have to install the Python 3.4 software package on your microscope-PC. The additional needed software-side packages and their last versions confirmed to work on Windows XP are: - `comtypes` – version 1.2.1 ([link](https://pypi.org/project/comtypes/1.2.1/#comtypes-1.2.1-py2.py3-none-any.whl)); -- `PyYAML` – version 5.1.2 ([link](https://pypi.org/project/PyYAML/5.1.2/#PyYAML-5.1.2-cp34-cp34m-win32.whl), - might require [updating pip](https://pypi.org/project/pip/19.1.1/#pip-19.1.1-py2.py3-none-any.whl)) -- `typing` – version 3.10.0.0 ([link](https://pypi.org/project/typing/3.10.0.0/#typing-3.10.0.0-py3-none-any.whl)) +- `pip` – version 19.1.1 ([link](https://pypi.org/project/pip/19.1.1/#pip-19.1.1-py2.py3-none-any.whl)); +- `PyYAML` – version 5.1.2 ([link](https://pypi.org/project/PyYAML/5.1.2/#PyYAML-5.1.2-cp34-cp34m-win32.whl)); +- `typing` – version 3.10.0.0 ([link](https://pypi.org/project/typing/3.10.0.0/#typing-3.10.0.0-py3-none-any.whl)); -The packages can be installed using pip, either automatically (if connected to the internet) or manually, by downloading, copying, and pointing to the wheel files linked above. Furthermore, the FEI com-scripting interface should be available on your TEM-PC. +If you furthermore intend to use `instamatic-tecnai-server` to interface an integrated camera: -## Usage +- `numpy` – version 1.15.4 ([link](https://pypi.org/project/numpy/1.15.4/#files)). + +The packages can be installed using pip, either automatically (if connected to the internet) or manually, by downloading, copying, and pointing to the wheel files linked above. Furthermore, the FEI com-scripting interface should be available on your TEM-PC. It is most convenient to create an installation in a custom virtual environment, e.g. using conda, on a removable or shared drive. -Microscope PC: This script -Camera PC (or other): [instamatic software](https://github.com/instamatic-dev/instamatic) +## Usage -The instamatic instance on the camera PC communicates with "Tecnai-Server"-software [over the network](https://instamatic.readthedocs.io/en/latest/network/). +On the Microscope PC, install this script. On the camera and/or support PC, install [instamatic software](https://github.com/instamatic-dev/instamatic). Follow example configuration 2 or 3 from the documentation. The instamatic GUI instance on the camera PC will communicate with "Tecnai-Server"-software [over the network](https://instamatic.readthedocs.io/en/latest/network/). -The "Tecnai-Server"-software is provided as a standard python-program. You can download and install the software in your chosen directory on the microscope-PC. After you have opened a MS-DOS command prompt you are navigating to the installation directory. The server will be started by the usual python invocation `py tem_server.py`. A corresponding `start.bat` -file provided in the installation directory can be adapted and used to run the command. +The "Tecnai-Server"-software is provided as a standard python-program. You can download and install the software in your chosen directory on the microscope-PC. After you have opened an MS-DOS command prompt you are navigating to the installation directory. The server will be started by the usual python invocation `py tem_server.py`. An integrated camera, if used, can be accessed using analogous `py cam_server.py`. A corresponding `start.bat` file provided in the installation directory can be adapted and used to run the command(s). The software will be configured by the .yaml-files in the `utils`-subdirectory. For instance the correct network address of your microscope-PC is set in the `settings.yaml` file. The magnification table of your TEM or the scripting interface `tecnai` are saved in the `microscope.yaml` file. -In our experimental setup the [instamatic software](https://github.com/instamatic-dev/instamatic) is installed on a separate PC (camera PC). In this case the configuration files of Instamatic must be adapted like in the server software. Especially the `interface="tecnai"`, the microscope, the network address and the flag `use_tem_server"` should be verified. Afterwards, the instamatic software should be starting without errors on your PC. You can try it out in an IPython shell if the TEMController-object has access to TEM. +In our experimental setup the [instamatic software](https://github.com/instamatic-dev/instamatic) is installed on a separate PC (camera PC). In this case the configuration files of Instamatic must be adapted like in the server software. Especially the `interface=tecnai`, the microscope, the network address and the flag `use_tem_server` should be verified. Afterward, the instamatic software should be starting without errors on your PC. You can use an IPython shell (`instamatic.controller`) to test if the TEMController-object has access to TEM. ## Credits -Thanks to Steffen Schmidt ([CUP, LMU München](https://www.cup.uni-muenchen.de/)) for providing this script. +Thanks to Steffen Schmidt ([CUP, LMU München](https://www.cup.uni-muenchen.de/)) for providing the original script. diff --git a/cam_server.py b/cam_server.py index 0779bea..f4d9008 100644 --- a/cam_server.py +++ b/cam_server.py @@ -1,4 +1,4 @@ -# BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE +# IF YOUR VENV DOES NOT WORK CORRECTLY, A BOILERPLATE LIKE THIS MAY BE REQUIRED # import sys # sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') # BOILERPLATE END @@ -6,12 +6,12 @@ import argparse import logging import queue -import sys import threading import time from instamaticServer.TEMController.camera import get_camera -from tem_server import DeviceServer, _conf, listen, setup_logging +from instamaticServer.utils.config import config +from tem_server import DeviceServer, listen, setup_logging BUFSIZE = 1024 @@ -27,8 +27,8 @@ class CamServer(DeviceServer): requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) stop_event = threading.Event() - host = _conf.default_settings['cam_server_host'] - port = _conf.default_settings['cam_server_port'] + host = config.default_settings['cam_server_host'] + port = config.default_settings['cam_server_port'] def __init__(self, name=None) -> None: super(CamServer, self).__init__(name=name) diff --git a/instamaticServer/TEMController/camera.py b/instamaticServer/TEMController/camera.py index ff296cb..4ab3d62 100644 --- a/instamaticServer/TEMController/camera.py +++ b/instamaticServer/TEMController/camera.py @@ -1,7 +1,6 @@ from instamaticServer.utils.config import config -_conf = config() _cam_interfaces = ('simulate', 'tecnai') __all__ = ['get_camera', 'get_camera_class'] @@ -26,8 +25,8 @@ def get_camera(name: str = None): if name in _cam_interfaces: interface = name else: - interface = _conf.camera.interface - name = _conf.default_settings['microscope'] + interface = config.camera.interface + name = config.default_settings['microscope'] cls = get_camera_class(interface=interface) tem = cls(name=name) diff --git a/instamaticServer/TEMController/microscope.py b/instamaticServer/TEMController/microscope.py index c1c6fb3..6e6cb3b 100644 --- a/instamaticServer/TEMController/microscope.py +++ b/instamaticServer/TEMController/microscope.py @@ -1,7 +1,6 @@ from instamaticServer.utils.config import config -_conf = config() _tem_interfaces = ('simulate', 'tecnai') __all__ = ['get_microscope', 'get_microscope_class'] @@ -26,8 +25,8 @@ def get_microscope(name: str = None): if name in _tem_interfaces: interface = name else: - interface = _conf.micr_interface - name = _conf.default_settings['microscope'] + interface = config.micr_interface + name = config.default_settings['microscope'] cls = get_microscope_class(interface=interface) tem = cls(name=name) diff --git a/instamaticServer/TEMController/simu_camera.py b/instamaticServer/TEMController/simu_camera.py index 04f7b8a..95d6cb1 100644 --- a/instamaticServer/TEMController/simu_camera.py +++ b/instamaticServer/TEMController/simu_camera.py @@ -55,8 +55,7 @@ def get_name(self) -> str: return self.name def load_defaults(self) -> None: - _conf = config() - for key, val in _conf.camera.__dict__.items(): + for key, val in config.camera.__dict__.items(): setattr(self, key, val) def get_image(self, exposure: Optional[float] = None, binsize: int = 1): diff --git a/instamaticServer/TEMController/simu_microscope.py b/instamaticServer/TEMController/simu_microscope.py index 1f2832d..ac2f458 100644 --- a/instamaticServer/TEMController/simu_microscope.py +++ b/instamaticServer/TEMController/simu_microscope.py @@ -3,7 +3,7 @@ import time from typing import Optional, Tuple, Union -from instamaticServer.utils.config import config +from instamaticServer.utils.config import Config from instamaticServer.utils.exceptions import TEMValueError from instamaticServer.utils.types import StagePositionTuple, float_deg, int_nm @@ -92,11 +92,11 @@ def __init__(self, name: str = None): self.MAX = MAX self.MIN = MIN - self._conf = config(self.name) + self.config = Config(self.name) self._mic_ranges = None - if self._conf.micr_interface == 'simulate': - self._mic_ranges = self._conf.micr_ranges + if self.config.micr_interface == 'simulate': + self._mic_ranges = self.config.micr_ranges self._HT = 200000 # V diff --git a/instamaticServer/TEMController/tecnai_camera.py b/instamaticServer/TEMController/tecnai_camera.py index b8076d0..0aeb292 100644 --- a/instamaticServer/TEMController/tecnai_camera.py +++ b/instamaticServer/TEMController/tecnai_camera.py @@ -61,8 +61,7 @@ def get_name(self) -> str: return self.name def load_defaults(self) -> None: - _conf = config() - for key, val in _conf.camera.__dict__.items(): + for key, val in config.camera.__dict__.items(): setattr(self, key, val) def get_image(self, exposure: Optional[float] = None, binsize: Optional[int] = None): diff --git a/instamaticServer/TEMController/tecnai_microscope.py b/instamaticServer/TEMController/tecnai_microscope.py index c130dca..d0f2855 100644 --- a/instamaticServer/TEMController/tecnai_microscope.py +++ b/instamaticServer/TEMController/tecnai_microscope.py @@ -6,7 +6,7 @@ from typing import Optional from instamaticServer.TEMController.tecnai_stage_thread import TecnaiStageThread -from instamaticServer.utils.config import config +from instamaticServer.utils.config import Config from instamaticServer.utils.exceptions import FEIValueError, TEMCommunicationError from instamaticServer.utils.singleton import Singleton from instamaticServer.utils.types import StagePositionTuple, float_deg, int_nm @@ -18,7 +18,7 @@ class TecnaiMicroscope(metaclass=Singleton): - """Python bindings to the Tecnai-G2 microscope using the COM scripting interface.""" + """Python bindings to the FEI microscope using the COM scripting interface.""" def __init__(self, name: str=None) -> None: comtypes.CoInitialize() @@ -45,10 +45,10 @@ def __init__(self, name: str=None) -> None: self.name = name - self._conf = config(self.name) + self.config = Config(self.name) self._mic_ranges = None - if self._conf.micr_interface == 'tecnai': - self._mic_ranges = self._conf.micr_ranges + if self.config.micr_interface == 'tecnai': + self._mic_ranges = self.config.micr_ranges self._rotation_speed = 1.0 self._tecnaiStage = TecnaiStageThread() #Thread für a-Movement @@ -170,7 +170,6 @@ def setStagePosition( #self._tem.Stage.GoToWithSpeed(pos, axis, 0.01) => 1grad in 4-5sec. - def setStageA(self, value: float=None, wait: bool=True) -> None: """Set the Stageposition alpha (A) in degrees.""" pos = self._tem.Stage.Position diff --git a/instamaticServer/serializer.py b/instamaticServer/serializer.py index 3b408b9..fcc7f4c 100644 --- a/instamaticServer/serializer.py +++ b/instamaticServer/serializer.py @@ -3,8 +3,7 @@ from instamaticServer.utils.config import config -_conf = config() -PROTOCOL = _conf.default_settings['tem_communication_protocol'] +PROTOCOL = config.default_settings['tem_communication_protocol'] # %timeit ctrl.stage.get() # - pickle: 287 µs ± 10.7 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each) @@ -50,5 +49,5 @@ def msgpack_dumper(data): loader = msgpack_loader dumper = msgpack_dumper else: - raise ValueError("No such protocol: %s" % (PROTOCOL)) + raise ValueError("No such protocol: %s" % PROTOCOL) diff --git a/instamaticServer/utils/config.py b/instamaticServer/utils/config.py index 8a7cd82..8ae1cc3 100644 --- a/instamaticServer/utils/config.py +++ b/instamaticServer/utils/config.py @@ -15,8 +15,7 @@ def dict_to_namespace(d: Dict) -> SimpleNamespace: return d -class config: - +class Config: def __init__(self, name:str=None): self.default_settings = self.settings() @@ -56,8 +55,11 @@ def load_camera_config(self) -> SimpleNamespace: return dict_to_namespace(yaml.safe_load(stream)) +config = Config() + + if __name__ == '__main__': - data = config() + data = Config() print(data.default_settings['microscope']) print(data.micr_ranges['Mh']) diff --git a/instamaticServer/utils/exceptions.py b/instamaticServer/utils/exceptions.py index 9371ff7..2800409 100644 --- a/instamaticServer/utils/exceptions.py +++ b/instamaticServer/utils/exceptions.py @@ -23,7 +23,7 @@ class TEMControllerError(Exception): 'TEMCommunicationError': TEMCommunicationError, 'JEOLValueError': JEOLValueError, 'FEIValueError': FEIValueError, - 'TEMControllerError ': TEMControllerError, + 'TEMControllerError': TEMControllerError, 'AttributeError': AttributeError, 'AssertionError': AssertionError, 'ConnectionError': ConnectionError, diff --git a/instamaticServer/utils/settings.yaml b/instamaticServer/utils/settings.yaml index 32fc3e8..3d07ff3 100644 --- a/instamaticServer/utils/settings.yaml +++ b/instamaticServer/utils/settings.yaml @@ -1,70 +1,21 @@ -microscope: titan -camera: ultrascan -calibration: simulate +microscope: simulate_tem # switch to e.g. 'tecnaiG2' after tests +camera: simulate_cam # switch to e.g. 'ultrascan' if desired # Global toggle to force simulated camera/microscope interface simulate: False data_directory: C:/instamatic -#flatfield: C:/instamatic/flatfield.tiff -flatfield: +flatfield: #C:/instamatic/flatfield.tiff # Run the TEM connection in a different process (recommended) use_tem_server: True -tem_server_host: '192.168.0.1' +tem_server_host: 'localhost' # '192.168.0.1' tem_server_port: 8088 tem_require_admin: False tem_communication_protocol: pickle # pickle, json, msgpack, yaml # Run the Camera connection in a different process -use_cam_server: True -cam_server_host: '192.168.0.1' +use_cam_server: False +cam_server_host: 'localhost' cam_server_port: 8087 cam_use_shared_memory: False - -# Submit collected data to an indexing server (CRED only) -use_indexing_server_exe: False -indexing_server_exe: 'instamatic.dialsserver.exe' -indexing_server_host: 'localhost' -indexing_server_port: 8089 -dials_script: 'E:/cctbx/dials_script.bat' - -# JEOL only, automatically set the rotation speed via Goniotool (instamatic.goniotool) -use_goniotool: False -goniotool_server_host: 'localhost' -goniotool_server_port: 8090 - -# For InsteaDMatic to control the rotation speed on a FEI/TFS system -fei_server_host: '192.168.12.1' -fei_server_port: 8091 - -# Automatically submit the data to an indexing server running in a VM (VirtualBox) -use_VM_server_exe: False -VM_server_exe: 'instamatic.VMserver.exe' -VM_server_host: 'localhost' -VM_server_port: 8092 -VM_ID: "Ubuntu 14.04.3" -VM_USERNAME: "lab6" -VM_PWD: "testtest" -VM_STARTUP_DELAY: 50 -VM_DESKTOP_DELAY: 20 -VM_SHARED_FOLDER: F:\SharedWithVM - -# Testing variables -cred_relax_beam_before_experiment: false -cred_track_stage_positions: false - -# Here the panels for the GUI can be turned on/off/reordered -modules: - - 'cred' - - 'cred_tvips' - - 'cred_fei' - - 'sed' - - 'autocred' - - 'red' - - 'machine_learning' - - 'ctrl' - - 'debug' - - 'about' - - 'console' - - 'io' diff --git a/instamaticServer/utils/titan.yaml b/instamaticServer/utils/titan.yaml new file mode 100644 index 0000000..11c74c2 --- /dev/null +++ b/instamaticServer/utils/titan.yaml @@ -0,0 +1,9 @@ +interface: tecnai +ranges: + D: [34, 42, 53, 68, 90, 115, 140, 175, 215, 265, 330, 420, 530, 680, 830, 1050, 1350, 1700, 2100, 2700, 3700] + LAD: [] + LM: [45, 58, 73, 89, 115, 145, 185, 235, 300, 380, 470, 600, 760, 950, 1200, 1550] + Mi: [] + SA: [3400, 4400, 5600, 7200, 8800, 11500, 14500, 18500, 24000, 30000, 38000, 49000, 61000, 77000, 100000, 130000, 165000, 215000, 265000, 340000, 430000, 550000, ] # 700000, 890000, 1150000, 1250000, - can these magnifications exist in multiple modes? + Mh: [700000, 890000, 1150000, 1250000, 960000, 750000, 600000, 470000, 360000, 285000, 225000, 175000, 145000, 115000, 89000, 66000, 52000, 41000, 32000, 26000, 21000, 8300, 6200, 3100, ] +wavelength: 0.019687 diff --git a/instamaticServer/utils/ultrascan.yaml b/instamaticServer/utils/ultrascan.yaml new file mode 100644 index 0000000..65f572e --- /dev/null +++ b/instamaticServer/utils/ultrascan.yaml @@ -0,0 +1,21 @@ +calib_beamshift: + gridsize: 5 + stepsize: 500 +calib_directbeam: + BeamShift: + gridsize: 5 + stepsize: 75 + DiffShift: + gridsize: 5 + stepsize: 300 +camera_rotation_vs_stage_xy: -2.24 +default_binsize: 4 +default_exposure: 0.1 +dimensions: [2048, 2048] +dynamic_range: 65535 +interface: tecnai +physical_pixelsize: 0.014 +possible_binsizes: [1, 2, 4] +stretch_amplitude: 0.0 +stretch_azimuth: 0.0 +dead_time: 0.5 diff --git a/requirements.txt b/requirements.txt index a567911..78690ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ comtypes PyYAML typing; python_version < '3.5' +# numpy # only used by camera server diff --git a/start.bat b/start.bat index f03427f..a1fa251 100644 --- a/start.bat +++ b/start.bat @@ -1,7 +1,10 @@ -cd Q:\DanielT\instamatic-tecnai-server -Q:\Malika\py\py34\python-3.4.4\python.exe Q:\DanielT\instamatic-tecnai-server\tem_server.py &:: -c +:: VARIANT 1 - IF YOU HAVE PROPERLY WORKING VENV +cd c:\instamatic\instamatic-tecnai-server &:: point to your installation directory +call .\venv\Scripts\activate.bat &:: uncomment and point to venv if using one +python tem_server.py &:: use `py`, `py3`, or `python` as needed +:: VARIANT 2 - CALL DIRECTLY, REQUIRES UNCOMMENTED BOILERPLATE +:: cd Q:\DanielT\instamatic-tecnai-server +:: Q:\Malika\py\py34\python-3.4.4\python.exe Q:\DanielT\instamatic-tecnai-server\tem_server.py -:: cd c:\instamatic\instamatic-tecnai-server &:: point to your installation directory -:: call .\venv\Scripts\activate.bat &:: uncomment and point to venv if using one -:: python tem_server.py &:: use `py`, `py3`, or `python` as needed +:: If using cam server, this file can be copied and also adapted for cam_server.py diff --git a/tem_server.py b/tem_server.py index 7e20740..52cf05e 100644 --- a/tem_server.py +++ b/tem_server.py @@ -1,4 +1,4 @@ -# BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE +# IF YOUR VENV DOES NOT WORK CORRECTLY, A BOILERPLATE LIKE THIS MAY BE REQUIRED # import sys # sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') # BOILERPLATE END @@ -24,7 +24,6 @@ logging.addLevelName(15, "EVAL") -_conf = config() _generators = {} BUFSIZE = 1024 TIMEOUT = 0.5 @@ -140,8 +139,8 @@ class TemServer(DeviceServer): requests = queue.Queue(maxsize=1) responses = queue.Queue(maxsize=1) stop_event = threading.Event() - host = _conf.default_settings['tem_server_host'] - port = _conf.default_settings['tem_server_port'] + host = config.default_settings['tem_server_host'] + port = config.default_settings['tem_server_port'] def handle(conn: socket.socket, server_type: Type[DeviceServer]) -> None: diff --git a/tests.py b/tests.py index a271e32..44a4db1 100644 --- a/tests.py +++ b/tests.py @@ -1,8 +1,9 @@ -# BOILERPLATE TO MAKE THINGS WORK WITH THE VENV ISSUE +# IF YOUR VENV DOES NOT WORK CORRECTLY, A BOILERPLATE LIKE THIS MAY BE REQUIRED # import sys # sys.path.insert(0, r'Q:\DanielT\instamatic-tecnai-server\venv\Lib\site-packages') # BOILERPLATE END +import abc import atexit import pickle import socket @@ -10,11 +11,15 @@ import time import unittest +from typing import Type + from instamaticServer.TEMController.simu_microscope import SimuMicroscope from instamaticServer.utils.config import SimpleNamespace, config, dict_to_namespace +from cam_server import CamServer +from tem_server import DeviceServer, TemServer, listen + _conf_dict = {'a': 1, 'b': {'c': 3, 'd': 4}} -_conf = config() PRECISION_NM = 250 PRECISION_DEG = 0.1 TIMEOUT = 30 @@ -32,9 +37,8 @@ def test_dict_to_namespace(self): self.assertEqual(ns.b, SimpleNamespace(**{'c': 3, 'd': 4})) def test_config(self): - global _conf - self.assertIn(_conf.micr_interface, {'tecnai', 'simulate'}) - self.assertIsInstance(_conf.camera, SimpleNamespace) + self.assertIn(config.micr_interface, {'tecnai', 'simulate'}) + self.assertIsInstance(config.camera, SimpleNamespace) class TestSerializers(unittest.TestCase): @@ -54,89 +58,70 @@ def test_msgpack_serializer(self): pass -class TestServer(unittest.TestCase): +class TestDeviceServer(unittest.TestCase, metaclass=abc.ABCMeta): + DeviceServer = None # type: Type[DeviceServer] + device_abbr = None # type: str @classmethod def setUpClass(cls): - from tem_server import CamServer, TemServer, listen - cls.tem_server = TemServer() - cls.tem_server.start() - cls.tem_listener = threading.Thread(target=listen, args=(TemServer,), name='tem_listener') - cls.tem_listener.start() - cls.threads = [cls.tem_server, cls.tem_listener] + cls.server = cls.DeviceServer() + cls.server.start() + ln = cls.device_abbr + '_listener' + cls.listener = threading.Thread(target=listen, args=(cls.DeviceServer,), name=ln) + cls.listener.start() + cls.threads = [cls.server, cls.listener] atexit.register(cls.tearDownClass) t0 = time.perf_counter() - while getattr(cls.tem_server, 'device', None) is None: + while getattr(cls.server, 'device', None) is None: if time.perf_counter() - t0 > 5: - raise RuntimeError("Server device did not initialize") + raise RuntimeError(cls.device_abbr.upper() + ' device did not initialize') time.sleep(0.05) - cls.cam_server = CamServer() - cls.cam_server.start() - cls.cam_listener = threading.Thread(target=listen, args=(CamServer,), name='cam_listener') - cls.cam_listener.start() - cls.threads.extend([cls.cam_server, cls.cam_listener]) - try: - cls.const = cls.tem_server.device._tem_constant + cls.const = cls.server.device._tem_constant except AttributeError: cls.const = None - tem_host = _conf.default_settings['tem_server_host'] - tem_port = _conf.default_settings['tem_server_port'] - cls.socket_tem = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - cls.socket_tem.connect((tem_host, tem_port)) - cls.socket_tem.settimeout(TIMEOUT) - - cam_host = _conf.default_settings['cam_server_host'] - cam_port = _conf.default_settings['cam_server_port'] - cls.socket_cam = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - cls.socket_cam.connect((cam_host, cam_port)) - cls.socket_cam.settimeout(TIMEOUT) + host = config.default_settings[cls.device_abbr + '_server_host'] + port = config.default_settings[cls.device_abbr + '_server_port'] + cls.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + cls.socket.connect((host, port)) + cls.socket.settimeout(TIMEOUT) @classmethod def tearDownClass(cls): - stop_program_event.set() + cls.server.stop_event.set() while cls.threads: thread = cls.threads.pop(0) thread.join(timeout=5) if thread.is_alive(): print('Thread %s did not exit' % thread.name) - if hasattr(cls, 'socket_tem') and cls.socket_tem: - try: - cls.socket_tem.close() - except Exception as e: - print("Error closing socket_tem:", e) - finally: - cls.socket_tem = None - - if hasattr(cls, 'socket_cam') and cls.socket_cam: + if hasattr(cls, 'socket') and cls.socket: try: - cls.socket_cam.close() + cls.socket.close() except Exception as e: - print("Error closing socket_cam:", e) + print('Error closing %s socket:' % cls.device_abbr, e) finally: - cls.socket_cam = None + cls.socket = None - @staticmethod - def socket_send(s, func, args = (), kwargs = None): + def send(self, func, args = (), kwargs = None): from instamaticServer.serializer import dumper, loader from instamaticServer.utils.exceptions import TEMCommunicationError, exception_list kwargs = kwargs or {} d = {'func_name': func, 'args': args, 'kwargs': kwargs} buffer_size = 1024 - if func in ('get_image', 'get_movie'): - buffer_size += 8 * _conf.camera.dimensions[0] * _conf.camera.dimensions[1] - s.sendall(dumper(d)) - response = s.recv(buffer_size) + if func in ('get_image', 'get_movie', '__gen_next__'): + buffer_size += 8 * config.camera.dimensions[0] * config.camera.dimensions[1] + self.socket.sendall(dumper(d)) + response = self.socket.recv(buffer_size) if response: for _ in range(100): # warrants all image/movie is collected try: status, data = loader(response) except (pickle.UnpicklingError, EOFError, ValueError): - response += s.recv(buffer_size) + response += self.socket.recv(buffer_size) time.sleep(0.01) else: break @@ -153,43 +138,42 @@ def socket_send(s, func, args = (), kwargs = None): else: raise ConnectionError('Unknown status code: %s' % status) - def tem_send(self, func, args = (), kwargs = None): - return self.socket_send(self.socket_tem, func, args, kwargs) - def cam_send(self, func, args = (), kwargs = None): - return self.socket_send(self.socket_cam, func, args, kwargs) +class TestTemServer(TestDeviceServer): + DeviceServer = TemServer + device_abbr = 'tem' def test_20_getHolderType(self): - r = self.tem_send('getHolderType') - if not isinstance(self.tem_server.device, SimuMicroscope): + r = self.send('getHolderType') + if not isinstance(self.server.device, SimuMicroscope): self.assertIsInstance(r, int) self.assertIn(r,list(self.const.StageHolderType.values())) def test_21_getStagePosition(self): - r = self.tem_send('getStagePosition') + r = self.send('getStagePosition') self.assertIsInstance(r, tuple) self.assertEqual(len(r), 5) def test_22_getStageSpeed(self): - r = self.tem_send('getStageSpeed') + r = self.send('getStageSpeed') self.assertEqual(r, 0.5) def test_23_is_goniotool_available(self): - r = self.tem_send('is_goniotool_available') + r = self.send('is_goniotool_available') self.assertEqual(r, False) def test_24_isAThreadAlive(self): - r = self.tem_send('isAThreadAlive') + r = self.send('isAThreadAlive') self.assertEqual(r, False) def test_25_isStageMoving(self): - r = self.tem_send('isStageMoving') + r = self.send('isStageMoving') self.assertEqual(r, False) def test_30_setStagePosition(self): p = (0, 0, 0, 0, 0) - self.tem_send('setStagePosition', p) - r = self.tem_send('getStagePosition') + self.send('setStagePosition', p) + r = self.send('getStagePosition') self.assertAlmostEqual(r[0], p[0], delta=PRECISION_NM) self.assertAlmostEqual(r[1], p[1], delta=PRECISION_NM) self.assertAlmostEqual(r[2], p[2], delta=PRECISION_NM) @@ -197,9 +181,9 @@ def test_30_setStagePosition(self): self.assertAlmostEqual(r[4], p[4], delta=PRECISION_DEG) def test_31_setStagePosition(self): - self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) - self.tem_send('setStagePosition', kwargs={'x': 10000, 'y': 10000}) - r = self.tem_send('getStagePosition') + self.send('setStagePosition', (0, 0, 0, 0, 0)) + self.send('setStagePosition', kwargs={'x': 10000, 'y': 10000}) + r = self.send('getStagePosition') self.assertAlmostEqual(r[0], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[1], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[2], 0, delta=PRECISION_NM) @@ -207,257 +191,262 @@ def test_31_setStagePosition(self): self.assertAlmostEqual(r[4], 0, delta=PRECISION_DEG) def test_32_setStagePosition(self): - self.tem_send('setStagePosition', (10000, 10000, 0, 0, 0)) - self.tem_send('getStagePosition') - self.tem_send('setStagePosition', kwargs={'z': 10000}) - r = self.tem_send('getStagePosition') + self.send('setStagePosition', (10000, 10000, 0, 0, 0)) + self.send('getStagePosition') + self.send('setStagePosition', kwargs={'z': 10000}) + r = self.send('getStagePosition') self.assertAlmostEqual(r[2], 10000, delta=PRECISION_NM) - self.tem_send('setStagePosition', kwargs={'z': 0}) - r = self.tem_send('getStagePosition') + self.send('setStagePosition', kwargs={'z': 0}) + r = self.send('getStagePosition') self.assertAlmostEqual(r[2], 0, delta=PRECISION_NM) - self.tem_send('setStagePosition', (10000, 10000, 10000, 0, 0)) - r = self.tem_send('getStagePosition') + self.send('setStagePosition', (10000, 10000, 10000, 0, 0)) + r = self.send('getStagePosition') self.assertAlmostEqual(r[0], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[1], 10000, delta=PRECISION_NM) self.assertAlmostEqual(r[2], 10000, delta=PRECISION_NM) def test_33_setStagePosition(self): - self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) - self.tem_send('setStagePosition', kwargs={'a': 10}) - r = self.tem_send('getStagePosition') + self.send('setStagePosition', (0, 0, 0, 0, 0)) + self.send('setStagePosition', kwargs={'a': 10}) + r = self.send('getStagePosition') self.assertAlmostEqual(r[3], 10, delta=PRECISION_DEG) def test_35_setStagePosition(self): - self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) + self.send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'x': 5000}) + self.send('setStagePosition', kwargs={'x': 5000}) t1 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.1}) + self.send('setStagePosition', kwargs={'x': 0, 'speed': 0.1}) t2 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'x': 5000, 'speed': 0.05}) + self.send('setStagePosition', kwargs={'x': 5000, 'speed': 0.05}) t3 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'x': 0, 'speed': 0.02}) + self.send('setStagePosition', kwargs={'x': 0, 'speed': 0.02}) t4 = time.perf_counter() - if not isinstance(self.tem_server.device, SimuMicroscope): + if not isinstance(self.server.device, SimuMicroscope): self.assertLess(t1 - t0, t2 - t1) self.assertLess(t2 - t1, t3 - t2) self.assertLess(t3 - t2, t4 - t3) def test_36_setStagePosition(self): - self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) + self.send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'a': 2}) + self.send('setStagePosition', kwargs={'a': 2}) t1 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'a': 0, 'speed': 0.1}) + self.send('setStagePosition', kwargs={'a': 0, 'speed': 0.1}) t2 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'a': 2, 'speed': 0.05}) + self.send('setStagePosition', kwargs={'a': 2, 'speed': 0.05}) t3 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'a': 0, 'speed': 0.02}) + self.send('setStagePosition', kwargs={'a': 0, 'speed': 0.02}) t4 = time.perf_counter() - if not isinstance(self.tem_server.device, SimuMicroscope): + if not isinstance(self.server.device, SimuMicroscope): self.assertLess(t2 - t1, t3 - t2) self.assertLess(t1 - t0, t2 - t1) self.assertLess(t3 - t2, t4 - t3) def test_38_setStagePosition(self): - self.tem_send('setStagePosition', (0, 0, 0, 0, 0)) + self.send('setStagePosition', (0, 0, 0, 0, 0)) t0 = time.perf_counter() - self.tem_send('setStagePosition', kwargs={'a': 30, 'wait': False}) + self.send('setStagePosition', kwargs={'a': 30, 'wait': False}) t1 = time.perf_counter() time.sleep(0.1) - self.tem_send('waitForStage') + self.send('waitForStage') t2 = time.perf_counter() q = {'a': 0, 'wait': True} - self.tem_send('setStagePosition', kwargs=q) + self.send('setStagePosition', kwargs=q) t3 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) self.assertLess(t1 - t0, t3 - t2) def test_40_setStageA(self): p = (0, ) - self.tem_send('setStageA', p) - r = self.tem_send('getStagePosition') + self.send('setStageA', p) + r = self.send('getStagePosition') self.assertAlmostEqual(r[3], p[0], delta=PRECISION_DEG) def test_46_setRotationSpeed(self): - self.tem_send('setStageA', kwargs={'value': 0}) + self.send('setStageA', kwargs={'value': 0}) t0 = time.perf_counter() - self.tem_send('setRotationSpeed', (1.0, )) - self.tem_send('setStageA', kwargs={'value': 5}) + self.send('setRotationSpeed', (1.0,)) + self.send('setStageA', kwargs={'value': 5}) t1 = time.perf_counter() - self.tem_send('setRotationSpeed', (0.1, )) - self.tem_send('setStageA', kwargs={'value': 0}) + self.send('setRotationSpeed', (0.1,)) + self.send('setStageA', kwargs={'value': 0}) t2 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) - self.tem_send('setRotationSpeed', (0.05, )) - self.tem_send('setStageA', kwargs={'value': 5}) + self.send('setRotationSpeed', (0.05,)) + self.send('setStageA', kwargs={'value': 5}) t3 = time.perf_counter() self.assertLess(t2 - t1, t3 - t2) - self.tem_send('setRotationSpeed', (1.0, )) - self.tem_send('setStageA', kwargs={'value': 0}) + self.send('setRotationSpeed', (1.0,)) + self.send('setStageA', kwargs={'value': 0}) def test_48_setStageA(self): - self.tem_send('setStageA', (0, )) + self.send('setStageA', (0,)) t0 = time.perf_counter() - self.tem_send('setStageA', kwargs={'value': 10, 'wait': False}) + self.send('setStageA', kwargs={'value': 10, 'wait': False}) t1 = time.perf_counter() time.sleep(0.1) - self.tem_send('waitForStage', kwargs={'delay': 0.01}) + self.send('waitForStage', kwargs={'delay': 0.01}) t2 = time.perf_counter() - self.tem_send('setStageA', kwargs={'value': 0, 'wait': True}) + self.send('setStageA', kwargs={'value': 0, 'wait': True}) t3 = time.perf_counter() self.assertLess(t1 - t0, t2 - t1) self.assertLess(t1 - t0, t3 - t2) def test_50_getGunShift(self): - self.tem_send('getGunShift') + self.send('getGunShift') def test_51_getHTValue(self): - self.tem_send('getHTValue') + self.send('getHTValue') def test_52_isBeamBlanked(self): - self.tem_send('isBeamBlanked') + self.send('isBeamBlanked') def test_53_getBeamAlignShift(self): - self.tem_send('getBeamAlignShift') + self.send('getBeamAlignShift') def test_54_getSpotSize(self): - self.tem_send('getSpotSize') + self.send('getSpotSize') def test_55_getBrightness(self): - self.tem_send('getBrightness') + self.send('getBrightness') def test_56_getBrightnessValue(self): - self.tem_send('getBrightnessValue') + self.send('getBrightnessValue') def test_57_getBeamShift(self): - self.tem_send('getBeamShift') + self.send('getBeamShift') def test_58_getBeamTilt(self): - self.tem_send('getBeamTilt') + self.send('getBeamTilt') def test_59_getCondensorLensStigmator(self): - self.tem_send('getCondensorLensStigmator') + self.send('getCondensorLensStigmator') def test_61_getScreenCurrent(self): - self.tem_send('getScreenCurrent') + self.send('getScreenCurrent') def test_62_isfocusscreenin(self): - r = self.tem_send('isfocusscreenin') + r = self.send('isfocusscreenin') self.assertIsInstance(r, bool) def test_63_getScreenPosition(self): - r = self.tem_send('getScreenPosition') + r = self.send('getScreenPosition') self.assertIn(r, {'up', 'down', ''}) def test_64_getDiffFocus(self): - func_mode = self.tem_send('getFunctionMode') - self.tem_send('setFunctionMode', ('diff', )) - r = self.tem_send('getDiffFocus') - self.tem_send('setFunctionMode', (func_mode,)) + func_mode = self.send('getFunctionMode') + self.send('setFunctionMode', ('diff',)) + r = self.send('getDiffFocus') + self.send('setFunctionMode', (func_mode,)) self.assertGreater(r, 0) self.assertLess(r, 65536) def test_65_getDiffFocusValue(self): - func_mode = self.tem_send('getFunctionMode') - self.tem_send('setFunctionMode', ('diff', )) - r = self.tem_send('getDiffFocusValue') - self.tem_send('setFunctionMode', (func_mode, )) - if not isinstance(self.tem_server.device, SimuMicroscope): + func_mode = self.send('getFunctionMode') + self.send('setFunctionMode', ('diff',)) + r = self.send('getDiffFocusValue') + self.send('setFunctionMode', (func_mode,)) + if not isinstance(self.server.device, SimuMicroscope): self.assertGreater(r, -1.0) self.assertLess(r, 1.0) def test_65_getFocus(self): - r = self.tem_send('getFocus') - if not isinstance(self.tem_server.device, SimuMicroscope): + r = self.send('getFocus') + if not isinstance(self.server.device, SimuMicroscope): self.assertGreater(r, -1.0) self.assertLess(r, 1.0) def test_67_FunctionMode(self): - r = self.tem_send('getFunctionMode') - self.tem_send('setFunctionMode', (r, )) + r = self.send('getFunctionMode') + self.send('setFunctionMode', (r,)) self.assertIn(r, ('lowmag', 'mag1', 'samag', 'mag2', 'diff')) def test_68_Magnification(self): - r = self.tem_send('getMagnification') - self.tem_send('setMagnification', (r, )) + r = self.send('getMagnification') + self.send('setMagnification', (r,)) self.assertIsInstance(r, (int, float)) def test_69_MagnificationIndex(self): - r = self.tem_send('getMagnificationIndex') - self.tem_send('setMagnificationIndex', (r, )) + r = self.send('getMagnificationIndex') + self.send('setMagnificationIndex', (r,)) self.assertIsInstance(r, int) def test_70_getDarkFieldTilt(self): - r = self.tem_send('getDarkFieldTilt') + r = self.send('getDarkFieldTilt') self.assertIsInstance(r[0], (int, float)) self.assertIsInstance(r[1], (int, float)) def test_71_getImageShift1(self): - r = self.tem_send('getImageShift1') + r = self.send('getImageShift1') self.assertIsInstance(r[0], (int, float)) self.assertIsInstance(r[1], (int, float)) def test_72_getImageShift2(self): - r = self.tem_send('getImageShift2') + r = self.send('getImageShift2') self.assertIsInstance(r[0], (int, float)) self.assertIsInstance(r[1], (int, float)) def test_73_getImageBeamShift(self): - r = self.tem_send('getImageBeamShift') + r = self.send('getImageBeamShift') self.assertIsInstance(r[0], (int, float)) self.assertIsInstance(r[1], (int, float)) def test_74_getDiffShift(self): - r = self.tem_send('getDiffShift') + r = self.send('getDiffShift') self.assertIsInstance(r[0], (int, float)) self.assertIsInstance(r[1], (int, float)) def test_76_getObjectiveLensStigmator(self): - r = self.tem_send('getObjectiveLensStigmator') + r = self.send('getObjectiveLensStigmator') self.assertIsInstance(r[0], (int, float)) self.assertIsInstance(r[1], (int, float)) def test_77_getIntermediateLensStigmator(self): - r = self.tem_send('getIntermediateLensStigmator') + r = self.send('getIntermediateLensStigmator') self.assertIsInstance(r[0], (int, float)) self.assertIsInstance(r[1], (int, float)) + +class TestCamServer(TestDeviceServer): + DeviceServer = CamServer + device_abbr = 'cam' + def test_90_get_binning(self): - r = self.cam_send('get_binning') + r = self.send('get_binning') self.assertIsInstance(r, int) def test_91_default_binsize(self): - r = self.cam_send('get_binning') - s = self.cam_send('default_binsize') + r = self.send('get_binning') + s = self.send('default_binsize') self.assertEqual(r, s) def test_92_dimensions(self): - r = self.cam_send('dimensions') + r = self.send('dimensions') self.assertIsInstance(r[0], int) def test_93_get_camera_dimensions(self): - r = self.cam_send('get_camera_dimensions') - s = self.cam_send('dimensions') + r = self.send('get_camera_dimensions') + s = self.send('dimensions') self.assertEqual(r, s) def test_94_get_image_dimensions(self): - r = self.cam_send('get_image_dimensions') - s = self.cam_send('get_camera_dimensions') + r = self.send('get_image_dimensions') + s = self.send('get_camera_dimensions') self.assertEqual(r[0], s[0]) self.assertEqual(r[1], s[1]) def test_96_get_image(self): - r = self.cam_send('get_image') - s = self.cam_send('get_image_dimensions') + r = self.send('get_image') + s = self.send('get_image_dimensions') self.assertEqual(len(r), s[0]) def test_97_get_image(self): - r = self.cam_send('get_image') - s = self.cam_send('get_image_dimensions') + r = self.send('get_image') + s = self.send('get_image_dimensions') self.assertEqual(len(r), s[0]) def test_98_get_movie(self): - r = self.cam_send('get_movie', (1, )) - s = self.cam_send('get_image_dimensions') + r = self.send('get_movie', (1, )) + s = self.send('get_image_dimensions') self.assertEqual(len(r[0]), s[0]) From 1f0cd5ebd57002df274f36421f8b07b85d70eaaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 14 Jan 2026 17:14:24 +0100 Subject: [PATCH 26/26] Remove unused import statements --- instamaticServer/TEMController/remote_movie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instamaticServer/TEMController/remote_movie.py b/instamaticServer/TEMController/remote_movie.py index 51d354f..c0d25a9 100644 --- a/instamaticServer/TEMController/remote_movie.py +++ b/instamaticServer/TEMController/remote_movie.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, Dict +from typing import Callable class RemoteMovie: