diff --git a/src/instamatic/camera/camera_gatan2.py b/src/instamatic/camera/camera_gatan2.py index 76dc1d8e..73c7999a 100644 --- a/src/instamatic/camera/camera_gatan2.py +++ b/src/instamatic/camera/camera_gatan2.py @@ -4,14 +4,16 @@ import logging import sys import time +import warnings from pathlib import Path -from typing import Tuple +from typing import Iterator, Optional, Tuple import numpy as np from instamatic import config from instamatic.camera.camera_base import CameraBase from instamatic.camera.gatansocket3 import GatanSocket +from instamatic.utils.deprecated import VisibleDeprecationWarning logger = logging.getLogger(__name__) @@ -32,7 +34,6 @@ def __init__(self, name: str = 'gatan2'): self.load_defaults() msg = f'Camera `{self.get_camera_name()}` ({self.name}) initialized' - # print(msg) logger.info(msg) atexit.register(self.release_connection) @@ -75,33 +76,71 @@ def write_tiffs(self) -> None: def get_image( self, - exposure=0.400, - binning=1, - processing='gain normalized', + exposure: float | None = None, + binsize: int | None = None, + *, + binning: int | None = None, + processing: str = 'gain normalized', + **kwargs, ) -> 'np.array': - """Acquire image through DM and return data as np array.""" + """Acquire image through DM and return data as np array. + + Notes + ----- + Instamatic's public API uses `binsize` everywhere (controller, + videostream, CLI). Historically this driver used `binning` instead, + which prevented GUI/Controller code from changing binning. + This implementation accepts both names. `binsize`, `binning`, + and `default_binsize` take precedence in this order. + Use of `binning` is deprecated and might be removed in the future. + """ + + exposure = exposure if exposure is not None else self.default_exposure + + if binning is not None: + msg = 'Argument `binning` is deprecated since v2.3.0. Use `binsize` instead.' + warnings.warn(msg, VisibleDeprecationWarning, stacklevel=2) + + if binsize is None: + binsize = binning if binning is not None else self.default_binsize - width, height = self.dimensions + if (p := getattr(self, 'possible_binsizes', [])) and binsize not in p: + msg = f'Cannot use {binsize=}, should be one of {self.possible_binsizes=}' + raise ValueError(msg) + + width, height = [int(wh) for wh in self.dimensions][:2] top = 0 left = 0 - bottom = height - right = width + bottom = int(height) + right = int(width) arr = self.g.get_image( processing=processing, height=height, width=width, - binning=binning, + binning=binsize, top=top, left=left, bottom=bottom, right=right, exposure=exposure, shutterDelay=0, + **kwargs, ) return arr + def get_movie(self, n_frames: int, *args, **kwargs) -> Iterator[np.ndarray]: + """Yield `n_frames` images by repeatedly calling `get_image`. + + The SerialEMCCD socket protocol seems to have no native streaming call, + so each frame here is a full round trip: re-arm, expose, read. There is + no guarantee of even dead time or frame spacing - this is a convenience + wrapper only. Provide a better implementation if you can. + """ + for _ in range(n_frames): + yield self.get_image(*args, **kwargs) + def acquire_image(self, **kwargs) -> 'np.array': """Acquire image through DM.""" return self.get_image(**kwargs) diff --git a/src/instamatic/camera/gatansocket3.py b/src/instamatic/camera/gatansocket3.py index 084608fe..264dea16 100644 --- a/src/instamatic/camera/gatansocket3.py +++ b/src/instamatic/camera/gatansocket3.py @@ -78,18 +78,22 @@ # lookup table of function name to function code, starting with 1 enum_gs = {x: y for (y, x) in enumerate(enum_gs, 1)} -# C "long" -> numpy "int_" +# NOTE: The SEMCCD (SerialEMCCD) socket protocol uses 32-bit signed integers for C "long" on Windows. +# Do NOT use np.int_ here: on 64-bit Python it is 64-bit and will break message sizes/offsets. ARGS_BUFFER_SIZE = 1024 MAX_LONG_ARGS = 16 MAX_DBL_ARGS = 8 MAX_BOOL_ARGS = 8 + +# Explicit 32-bit, little-endian integer used by the SEMCCD socket protocol (Windows C long). +C_LONG = np.dtype(' np.ndarray: - """Convert the string to a 1D np array of dtype (default np.int_ - C long) - with numpy2-save padding to ensure length is a multiple of dtype.itemsize. - """ +def string_to_longarray(string: str, *, dtype: np.dtype = C_LONG) -> np.ndarray: + """Convert the string to a 1D np array of dtype (default C_LONG (32-bit C + long on Windows)) with numpy2-save padding to ensure length is a multiple + of dtype.itemsize.""" s_bytes = string.encode('utf-8') dtype_size = np.dtype(dtype).itemsize if extra := len(s_bytes) % dtype_size: @@ -104,8 +108,8 @@ class Message: optional long array. """ - def __init__(self, longargs=[], boolargs=[], dblargs=[], longarray=[]): - # Strings are packaged as long array using np.frombuffer(buffer,np.int_) + def __init__(self, longargs=(), boolargs=(), dblargs=(), longarray=()): + # Strings are packaged as long array using np.frombuffer(buffer, C_LONG) # and can be converted back with longarray.tobytes() # add final longarg with size of the longarray if len(longarray): @@ -113,11 +117,11 @@ def __init__(self, longargs=[], boolargs=[], dblargs=[], longarray=[]): longargs.append(len(longarray)) self.dtype = [ - ('size', np.intc), - ('longargs', np.int_, (len(longargs),)), + ('size', np.int32), + ('longargs', C_LONG, (len(longargs),)), ('boolargs', np.int32, (len(boolargs),)), ('dblargs', np.double, (len(dblargs),)), - ('longarray', np.int_, (len(longarray),)), + ('longarray', C_LONG, (len(longarray),)), ] self.array = np.zeros((), dtype=self.dtype) self.array['size'] = self.array.data.itemsize @@ -127,10 +131,10 @@ def __init__(self, longargs=[], boolargs=[], dblargs=[], longarray=[]): self.array['longarray'] = longarray # create numpy arrays for the args and array - # self.longargs = np.asarray(longargs, dtype=np.int_) + # self.longargs = np.asarray(longargs, dtype=C_LONG) # self.dblargs = np.asarray(dblargs, dtype=np.double) # self.boolargs = np.asarray(boolargs, dtype=np.int32) - # self.longarray = np.asarray(longarray, dtype=np.int_) + # self.longarray = np.asarray(longarray, dtype=C_LONG) def pack(self): """Serialize the data.""" @@ -178,7 +182,7 @@ def __init__(self, host='', port=None): if port is not None: self.port = port elif 'SERIALEMCCD_PORT' in os.environ: - self.port = os.environ['SERIALEMCCD_PORT'] + self.port = int(os.environ['SERIALEMCCD_PORT']) else: raise ValueError( 'Must specify a port to GatanSocket instance, or set environment variable SERIALEMCCD_PORT' @@ -234,12 +238,30 @@ def hasScriptFunction(self, name): return result > 0.0 def connect(self): - # recommended by Gatan to use localhost IP to avoid using tcp - self.sock = socket.create_connection(('127.0.0.1', self.port)) + """Connect to SERIALEMCCD socket; use 127.0.0.1 when DM and client are + on the same PC; otherwise set `host` to the DM PC IP/hostname.""" + + host = self.host or '127.0.0.1' + self.sock = socket.create_connection((host, self.port)) + + try: # Prevent indefinite hangs on recv() + self.sock.settimeout(float(os.environ.get('SERIALEMCCD_TIMEOUT', 10))) + except Exception: + pass + try: + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + except Exception: + pass def disconnect(self): - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() + try: + self.sock.shutdown(socket.SHUT_RDWR) + except Exception: + pass + try: + self.sock.close() + except Exception: + pass def reconnect(self): self.disconnect() @@ -247,7 +269,14 @@ def reconnect(self): @logwrap def send_data(self, data): - return self.sock.sendall(data) + try: + return self.sock.sendall(data) + except (TimeoutError, socket.timeout, OSError) as exc: + try: # DM/SEMCCD can stop reading when busy; reconnect and retry once. + self.reconnect() + return self.sock.sendall(data) + except Exception: + raise exc @logwrap def recv_data(self, n): @@ -346,22 +375,11 @@ def SetK2Parameters( funcCode = enum_gs['GS_SetK2Parameters'] self.save_frames = saveFrames - longarray = string_to_longarray(filt + '\0', dtype=np.int_) # filter name + longarray = string_to_longarray(filt + '\0', dtype=C_LONG) # filter name - longs = [ - funcCode, - readMode, - hardwareProc, - ] - bools = [ - doseFrac, - alignFrames, - saveFrames, - ] - doubles = [ - scaling, - frameTime, - ] + longs = [funcCode, readMode, hardwareProc] + bools = [doseFrac, alignFrames, saveFrames] + doubles = [scaling, frameTime] message_send = Message( longargs=longs, boolargs=bools, dblargs=doubles, longarray=longarray @@ -401,7 +419,7 @@ def SetupFileSaving( longs = [enum_gs['GS_SetupFileSaving'], rotationFlip] dbls = [pixelSize] bools = [filePerImage] - longarray = string_to_longarray(dirname + '\0' + rootname + '\0', dtype=np.int_) + longarray = string_to_longarray(dirname + '\0' + rootname + '\0', dtype=C_LONG) message_send = Message( longargs=longs, boolargs=bools, dblargs=dbls, longarray=longarray ) @@ -481,31 +499,58 @@ def AlignEnergyFilterZeroLossPeak(self): script = f' if ( {func}() ) {{ {wait} Exit(1.0); }} else {{ Exit(-1.0); }}' return self.ExecuteGetDoubleScript(script) + # alias for GetImage, get_image was for some reason undefined in gatan_camera2.py + def get_image(self, *args, **kwargs): + return self.GetImage(*args, **kwargs) + @logwrap def GetImage( self, - processing, - height, - width, - binning, - top, - left, - bottom, - right, - exposure, # s - shutterDelay=0, # ms + processing: str, + height: int, + width: int, + binning: int, + top: int, + left: int, + bottom: int, + right: int, + exposure: float, # s + shutterDelay: int = 0, # ms ): """ processing : str Must be one of 'dark', 'unprocessed', 'dark subtracted', 'gain normalized' + + Notes + ----- + The SEMCCD plugin expects width/height and ROI coordinates in *binned* + pixels for CM_SetBinnedReadArea. If we pass full-frame dimensions with + binning>1, DM will throw an exception, and the plugin will return zeros. """ + binning = int(binning) if binning is not None else 1 + + # Convert ROI from unbinned to binned coordinates when binning>1. + # We interpret (top,left,bottom,right) as an unbinned ROI in pixels + # and downscale ROI to binned pixel coordinates (truncate to integers). + if binning > 1: + roi_w = right - left + roi_h = bottom - top + if roi_w <= 0 or roi_h <= 0: + raise ValueError(f'Invalid ROI: {left=}, {top=}, {right=}, {bottom=}') + left //= binning + top //= binning + width = roi_w // binning + height = roi_h // binning + right = left + width + bottom = top + height + arrSize = width * height # TODO: need to figure out what these should be shutter = 0 divideBy2 = 0 - corrections = 0 + corrections = -1 settling = 0.0 # prepare args for message @@ -561,32 +606,57 @@ def GetImage( longargs = message_recv.array['longargs'] if longargs[0] < 0: return 1 - arrSize = longargs[1] - width = longargs[2] - height = longargs[3] - numChunks = longargs[4] + arrSize = int(longargs[1]) + width = int(longargs[2]) + height = int(longargs[3]) + numChunks = int(longargs[4]) bytesPerPixel = 2 numBytes = arrSize * bytesPerPixel - chunkSize = (numBytes + numChunks - 1) / numChunks - imArray = np.zeros((height, width), np.ushort) + + # Defensive checks: if the plugin returns invalid dimensions, fail fast with diagnostics. + if width <= 0 or height <= 0 or arrSize <= 0: + raise RuntimeError( + f'Invalid image metadata from SEMCCD plugin: ' + f'{arrSize=}, {width=}, {height=}, {numChunks=}' + ) + if numChunks <= 0: + raise RuntimeError( + f'Invalid chunk count from SEMCCD plugin: ' + f'{numChunks=} ({arrSize=}, {width=}, {height=})' + ) + + # Use integer chunk size (ceil division). Float chunk sizes can cause recv() to misbehave. + chunkSize = (numBytes + numChunks - 1) // numChunks + + # IMPORTANT (Python 3.11+): avoid numpy/memoryview slice assignment into a multi-dim buffer. + # Accumulate raw bytes into a 1D bytearray, then view as uint16. + buf = bytearray(numBytes) + mv = memoryview(buf) received = 0 remain = numBytes + for chunk in range(numChunks): # send chunk handshake for all but the first chunk if chunk: message_send = Message(longargs=(enum_gs['GS_ChunkHandshake'],)) self.ExchangeMessages(message_send) + thisChunkSize = min(remain, chunkSize) - chunkReceived = 0 chunkRemain = thisChunkSize while chunkRemain: new_recv = self.recv_data(chunkRemain) + if not new_recv: + msg = f'Socket closed while receiving image data ({received=}/{numBytes} bytes)' + raise ConnectionError(msg) len_recv = len(new_recv) - imArray.data[received : received + len_recv] = new_recv - chunkReceived += len_recv + mv[received : received + len_recv] = new_recv chunkRemain -= len_recv remain -= len_recv received += len_recv + + # Convert bytes -> uint16 image + imArray = np.frombuffer(buf, dtype=np.uint16, count=arrSize) + imArray = imArray.reshape((height, width)) return imArray def ExecuteSendCameraObjectionFunction(self, function_name, camera_id=0): @@ -665,9 +735,9 @@ def ExecuteScript( recv_dblargs_init=(0.0,), recv_longarray_init=None, ): - """Send the command string as a 1D longarray of np.int_ dtype.""" + """Send the command string as a 1D longarray of C_LONG dtype.""" funcCode = enum_gs['GS_ExecuteScript'] - longarray = string_to_longarray(command_line + '\0', dtype=np.int_) + longarray = string_to_longarray(command_line + '\0', dtype=C_LONG) message_send = Message( longargs=(funcCode,), boolargs=(select_camera,), longarray=longarray )