fix antenna order post refactor

Used to store the protobuffer messages into files, so that they can be
plotted as part of the dissertation

examples/motion_detector.py

@@ -0,0 +1,72 @@
+"""Motion Detection example
+This example shows how to use the CSI framework to connect to a FeitCSI host and
+detect changes in the environment
+"""
+
+from queue import Queue
+
+import numpy as np
+import numpy.typing as npt
+
+from where_fi.application import CSIApplication
+from where_fi.collection.ingest import RealtimeCSIProducer
+
+# Connect to a FeitCSI host
+producer = RealtimeCSIProducer()
+app = CSIApplication(producer)
+
+
+# Stores historical data for each receiving antenna, for each subcarrier
+historical: dict[tuple[int, int], Queue[np.complex64]] = {}
+
+MAGN_THRESHOLD = 20
+PHASE_THRESHOLD = 0.5
+QUEUE_SIZE = 2000
+
+
+@app.on_process
+def _(sample: npt.NDArray[np.complex64]) -> None:
+    """
+    Process the CSI data and detect changes in the environment.
+
+    This function is called by the framework at a fixed interval, with the latest CSI
+    sample received.
+
+    It is used to detect changes in the environment caused by motion, by comparing each
+    entry in the matrix with a moving average
+    """
+    change = False
+    for antenna in range(sample.shape[1]):
+        for subcarrier in range(sample.shape[0]):
+            # Get the current subcarrier data
+            current = sample[subcarrier, antenna, 0]
+
+            # Get the historical data for this antenna and subcarrier
+            if (antenna, subcarrier) not in historical:
+                historical[(antenna, subcarrier)] = Queue(maxsize=QUEUE_SIZE)
+            historical_data = historical[(antenna, subcarrier)]
+
+            # Calculate the average of the historical data for this antenna and
+            # subcarrier
+            mean = np.mean(historical_data.queue)
+
+            # If we have enough historical data, compare it with the current data
+            if historical_data.full():
+                historical_data.get()
+
+            # Add the current sample to the historical data
+            historical_data.put(current)
+
+            # Compare the current data with the historical data
+            if (
+                np.abs(mean - current) > MAGN_THRESHOLD
+                or np.abs(np.angle(mean) - np.angle(current)) > PHASE_THRESHOLD
+            ):
+                change = True
+    if change:
+        print("Motion detected!")
+    else:
+        print("No motion detected!")
+
+
+app.start()

examples/motion_detector_home_assistant.py

@@ -0,0 +1,109 @@
+"""Motion Detection example
+This example shows how to use the CSI framework to connect to a FeitCSI host and
+detect changes in the environment
+
+Once motion is detected, the application will log the change to Home Assistant
+through an HTTP request.
+"""
+
+import os
+from queue import Queue
+
+import numpy as np
+import numpy.typing as npt
+import requests
+
+from where_fi.application import CSIApplication
+from where_fi.collection.ingest import RealtimeCSIProducer
+
+# Connect to a FeitCSI host
+producer = RealtimeCSIProducer()
+app = CSIApplication(producer)
+
+
+class HomeAssistantBinarySensor:
+    """
+    Represents a binary sensor in Home Assistant.
+
+    Uses the HTTP API [1] to update the state of the sensor.
+
+    [1] - https://www.home-assistant.io/integrations/http/#binary-sensor
+    """
+
+    def __init__(self, id: str, name: str) -> None:
+        self.id = id
+        self.name = name
+
+        BASE_URL = os.getenv("HOME_ASSISTANT_URL")
+        API_KEY = os.getenv("HOME_ASSISTANT_API_KEY")
+
+        self.url = f"{BASE_URL}/api/states/binary_sensor.{self.id}"
+        self.headers = {"Authorization": f"Bearer {API_KEY}"}
+
+        self.state = False
+
+    def update(self, state: bool) -> None:
+        if self.state == state:
+            return
+
+        self.state = state
+        data = {
+            "state": "on" if state else "off",
+            "attributes": {"friendly_name": self.name},
+        }
+        requests.post(self.url, json=data, headers=self.headers)
+
+
+# Stores historical data for each receiving antenna, for each subcarrier
+historical: dict[tuple[int, int], Queue[np.complex64]] = {}
+sensor = HomeAssistantBinarySensor("motion_detector", "Motion Detector")
+
+MAGN_THRESHOLD = 20
+PHASE_THRESHOLD = 0.5
+QUEUE_SIZE = 2000
+
+
+@app.on_process
+def _(sample: npt.NDArray[np.complex64]) -> None:
+    """
+    Process the CSI data and detect changes in the environment.
+
+    This function is called by the framework at a fixed interval, with the latest CSI
+    sample received.
+
+    It is used to detect changes in the environment caused by motion, by comparing each
+    entry in the matrix with a moving average
+    """
+    change = False
+    for antenna in range(sample.shape[1]):
+        for subcarrier in range(sample.shape[0]):
+            # Get the current subcarrier data
+            current = sample[subcarrier, antenna, 0]
+
+            # Get the historical data for this antenna and subcarrier
+            if (antenna, subcarrier) not in historical:
+                historical[(antenna, subcarrier)] = Queue(maxsize=QUEUE_SIZE)
+            historical_data = historical[(antenna, subcarrier)]
+
+            # Calculate the average of the historical data for this antenna and
+            # subcarrier
+            mean = np.mean(historical_data.queue)
+
+            # If we have enough historical data, compare it with the current data
+            if historical_data.full():
+                historical_data.get()
+
+            # Add the current sample to the historical data
+            historical_data.put(current)
+
+            # Compare the current data with the historical data
+            if (
+                np.abs(mean - current) > MAGN_THRESHOLD
+                or np.abs(np.angle(mean) - np.angle(current)) > PHASE_THRESHOLD
+            ):
+                change = True
+
+    sensor.update(change)
+
+
+app.start()

pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
     "pydantic>=2.10.6",
     "torch",
     "grpcio>=1.70.0",
+    "requests>=2.32.3",
 ]
 
 [build-system]
@@ -50,6 +51,9 @@ packages = ["where_fi"]
 
 [dependency-groups]
 dev = [
+    "grpc-stubs>=1.53.0.5",
     "grpcio-tools>=1.70.0",
+    "matplotlib-stubs>=0.1.0",
     "protoletariat>=3.3.9",
+    "types-requests>=2.32.0.20250328",
 ]

where_fi/__main__.py

@@ -2,11 +2,10 @@ import logging
 
 from . import cli
 
-
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s %(name)-50s %(levelname)-8s %(message)s",
 )
 
 if __name__ == "__main__":
-    cli.app()
+    cli.cli()

where_fi/application.py

@@ -0,0 +1,217 @@
+import logging
+import multiprocessing as mp
+import threading
+import time
+from typing import Any, Callable, NamedTuple
+
+import numpy.typing as npt
+
+from where_fi.collection import CSIMatrix, ingest
+from where_fi.collection.csi_frame import CSI
+from where_fi.collection.protocols import CSIProducer, MergedCSI
+from where_fi.config import config
+from where_fi.processing.preprocess import Preprocessor
+from where_fi.visualise import server as visualise
+
+
+class Receiver(NamedTuple):
+    ip: ingest.Host
+    receiver: ingest.FeitReceiver
+    thread: threading.Thread
+
+
+class Scheduler:
+    """
+    A simple scheduler that runs a function at a given rate.
+    """
+
+    def __init__(self, rate: float, func: Callable[[], None]) -> None:
+        self.rate = rate
+        self.func = func
+        self.active = True
+
+    def run(self) -> None:
+        """
+        Run the scheduler in a loop, calling the function at the given rate.
+        """
+        while self.active:
+            self.func()
+            time.sleep(1 / self.rate)
+
+    def start(self) -> None:
+        """
+        Start the scheduler in a separate thread.
+        """
+        self.thread = threading.Thread(target=self.run)
+        self.thread.start()
+
+    def stop(self) -> None:
+        """
+        Stop the scheduler.
+        """
+        self.active = False
+
+
+class CSIApplication:
+    """
+    A high-level application that manages CSI data ingestion and processing.
+
+    This allows a simple interface to be used for the main CLI logic, without having to
+    deal with the threads and queues directly.
+
+    It can be used using decortors to register callbacks for different stages of the
+    processing pipeline:
+
+        - `on_pre_merge`: Called with the raw CSI data (including headers) from each
+                          receiver, before it is merged into a single matrix.
+        - `on_sample`: Called once per sample with the merged CSI data.
+        - `on_process`: Called at the processing sample rate, without any data (this is
+                        mostly used as a scheduler).
+    """
+
+    def __init__(self, producer: CSIProducer, visualise_raw: bool = False) -> None:
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+
+        self.producer = producer
+
+        self.webapp_queue: "mp.Queue[visualise.VisualiserData]" = mp.Queue(
+            config.processing_sample_rate
+        )
+        self.webapp = visualise.Webapp()
+        self.webapp_thread = threading.Thread(
+            target=self.webapp.start, args=(self.webapp_queue,)
+        )
+        self.visualise_raw = visualise_raw
+
+        self.pre_merge_callback = None
+        self.raw_csi_callback = None
+        self.preprocessed_csi_callback = None
+        self.processing_callback = None
+
+        self.preprocessor = Preprocessor()
+
+    def visualise_data(
+        self, data: npt.NDArray[Any], dtype: visualise.figures.Figure
+    ) -> None:
+        """
+        Update a visualisation with the given data. The available visualisations are as
+        per visualise.server.all_figures.
+        """
+        if not self.webapp_queue.full():
+            self.webapp_queue.put(visualise.VisualiserData(data, dtype))
+
+    def on_raw(self, func: Callable[[CSIMatrix], None]) -> Callable[[CSIMatrix], None]:
+        """
+        Decorator to register a callback for the sample preprocessing.
+        """
+
+        def decorator(data: CSIMatrix) -> None:
+            func(data)
+
+        self.raw_csi_callback = decorator
+        return decorator
+
+    def on_sample(
+        self, func: Callable[[CSIMatrix], None]
+    ) -> Callable[[CSIMatrix], None]:
+        """
+        Decorator to register a callback for the sample preprocessing.
+        """
+
+        def decorator(data: CSIMatrix) -> None:
+            self.visualise_data(data, visualise.figures.Figure.RAW_CSI)
+            func(data)
+
+        self.preprocessed_csi_callback = decorator
+        return decorator
+
+    def on_pre_merge(
+        self, func: Callable[[dict[ingest.Host, CSI]], None]
+    ) -> Callable[[dict[ingest.Host, CSI]], None]:
+        """
+        Decorator to register a callback for the samples before merging.
+        """
+        self.pre_merge_callback = func
+        return func
+
+    def on_process(
+        self, func: Callable[[CSIMatrix], None]
+    ) -> Callable[[CSIMatrix], None]:
+        """
+        Decorator to register a callback for the sample processing.
+        """
+        self.processing_callback = func
+        return func
+
+    def process_sample(self, sample: MergedCSI) -> None:
+        """Data pipeline for processing a sample.
+        This is the entry point for a sample being received from the producer. Depending
+        on the callbacks which are registered, this will call the appropriate functions
+        to pre-process and visualise the sample.
+        """
+        if self.visualise_raw:
+            self.visualise_data(sample.matrix, visualise.figures.Figure.RAW_CSI)
+        if self.raw_csi_callback is not None:
+            self.raw_csi_callback(sample.matrix)
+        if self.pre_merge_callback is not None:
+            self.pre_merge_callback(sample.frames)
+        processed = self.preprocessor.preprocess(sample.matrix)
+        if self.visualise_raw:
+            self.visualise_data(processed, visualise.figures.Figure.PROCESSED_CSI)
+        if self.preprocessed_csi_callback is not None:
+            self.preprocessed_csi_callback(processed)
+
+    def listen(self) -> None:
+        """
+        Listen for incoming data from the selected ingestor, and call the appropriate
+        callback.
+        """
+        data_gen = self.producer()
+        while True:
+            try:
+                sample = next(data_gen)
+                self.process_sample(sample)
+            except StopIteration:
+                break
+            except Exception as e:
+                self.logger.error(f"Error processing sample: {e}", exc_info=True)
+                break
+
+    def start(self) -> None:
+        # for receiver in self.receivers:
+        #     receiver.thread.start()
+        self.buffer_thread = threading.Thread(
+            target=self.listen,
+        )
+        self.buffer_thread.start()
+        self.webapp_thread.start()
+
+        if self.processing_callback is not None:
+
+            def get_proc_sample() -> None:
+                sample = self.preprocessor.last_sample
+                if sample is not None and self.processing_callback is not None:
+                    self.processing_callback(sample)
+                else:
+                    self.logger.warning("No samples to process")
+
+            self.scheduler = Scheduler(config.processing_sample_rate, get_proc_sample)
+            self.scheduler.start()
+        try:
+            self.webapp_thread.join()
+        except KeyboardInterrupt:
+            self.stop()
+
+    def stop(self) -> None:
+        self.logger.info("Stopping application")
+
+        self.producer.stop()
+
+        if hasattr(self, "scheduler"):
+            self.scheduler.stop()
+            self.scheduler.thread.join()
+        self.buffer_thread.join()
+
+        self.webapp.active = False
+        self.webapp_thread.join()
+        self.logger.info("Application stopped")

where_fi/cli/__init__.py

@@ -1,24 +1,26 @@
 import logging
-import multiprocessing as mp
-from typing import Any
+from queue import Queue
+from typing import cast
 
 import numpy as np
 import numpy.typing as npt
 import torch
 import typer
 
+from where_fi.collection import CSIMatrix
+
+from ..application import CSIApplication
 from ..config import config
 from ..processing.aoa import AoA
-from ..processing.preprocess import Preprocessor
 from ..visualise import server as visualise
 from . import file, globals
 
-app = typer.Typer(callback=globals.main)
+cli = typer.Typer(callback=globals.main)
 logger = logging.getLogger(__name__)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 
-@app.command()
+@cli.command()
 def antennas() -> None:
     """Utility to help determine the order in which antennas are plugged in
 
@@ -29,38 +31,66 @@ def antennas() -> None:
     """
     from ..utils import antenna_order
 
-    if not globals.is_live:
-        raise ValueError("This command only works with live data")
+    if not globals.csi_producer.is_live:
+        raise NotImplementedError("This command only works with live data")
 
     antenna_order.main()
 
 
-@app.command()
+@cli.command()
 def heatmap() -> None:
-    preprocessor = Preprocessor()
+    app = CSIApplication(globals.csi_producer, visualise_raw=True)
     aoa = AoA()
 
-    # Start webapp in background process
-    webapp_queue: "mp.Queue[visualise.VisualiserData]" = mp.Queue(config.sample_rate)
-    webapp = mp.Process(target=visualise.start, args=(webapp_queue,))
-    webapp.start()
-
-    def visualise_data(data: npt.NDArray[Any], dtype: visualise.figures.Figure) -> None:
-        if not webapp_queue.full():
-            webapp_queue.put(visualise.VisualiserData(data, dtype))
-
-    def callback(antenna_data: npt.NDArray[np.complex64]) -> None:
-        logger.info(f"Got final CSI data with shape {antenna_data.shape}")
-        visualise_data(antenna_data, visualise.figures.Figure.RAW_CSI)
-        processed = preprocessor.preprocess(antenna_data, visualiser=visualise_data)
-        visualise_data(processed, visualise.figures.Figure.PROCESSED_CSI)
-        logger.info(f"Processed CSI data with shape {processed.shape}")
-        processed_tensor = torch.tensor(processed, device=device)
+    @app.on_sample
+    def _(sample: npt.NDArray[np.complex64]) -> None:
+        processed_tensor = torch.tensor(sample, device=device)
         aoa.update(processed_tensor)
-        aoa.heatmap(visualiser=visualise_data)
+        aoa.heatmap(visualiser=app.visualise_data)
 
-    globals.csi_producer(csi_callback=callback)
+    app.start()
     logger.info("Finished processing CSI data")
 
 
-app.add_typer(file.app, name="file", help="Commands for working with CSI files")
+@cli.command()
+def phase_analysis(
+    subcarriers: list[int] = [0], rx_antenna: int = 0, tx_antenna: int = 0
+) -> None:
+    """
+    Visualise the phase information in the CSI data received from the antennas.
+
+    The data goes through the same preprocessing steps as the heatmap command, but
+    instead of going through the AoA estimation, we simply analyse the phase of the
+    selected subcarrier and antenna.
+    """
+
+    app = CSIApplication(globals.csi_producer)
+
+    if isinstance(subcarriers, int):
+        subcarriers = [subcarriers]
+    print("Starting phase analysis on subcarriers: ", subcarriers)
+
+    subcarrier_phase: dict[int, Queue[float]] = {
+        x: Queue(config.collection_sample_rate) for x in subcarriers
+    }
+
+    @app.on_sample
+    def _(sample: CSIMatrix) -> None:
+        for subcarrier in subcarriers:
+            phase = cast(float, np.angle(sample[subcarrier, rx_antenna, tx_antenna]))
+            if subcarrier_phase[subcarrier].full():
+                subcarrier_phase[subcarrier].get()
+            subcarrier_phase[subcarrier].put(phase)
+
+    @app.on_process
+    def _(_: CSIMatrix) -> None:
+        logger.info("Updating phase visualisation")
+        app.visualise_data(
+            np.array([x.queue for x in subcarrier_phase.values()]),
+            visualise.figures.Figure.PHASE_ANALYSIS,
+        )
+
+    app.start()
+
+
+cli.add_typer(file.app, name="file", help="Commands for working with CSI files")

where_fi/cli/globals.py

@@ -1,18 +1,15 @@
-from functools import partial
 from pathlib import Path
 
 from .. import collection
 from ..collection import file, ingest
 
-csi_producer: collection.CSIProducer = collection.noop
+csi_producer: collection.CSIProducer = collection.NoopCSIProducer()
 is_live = True
 
 
 def main(from_file: Path | None = None) -> None:
     global csi_producer, is_live
     if from_file:
-        csi_producer = partial(file.start_processing, file_path=from_file)
-        is_live = False
+        csi_producer = file.FileCSIPRoducer(path=from_file)
     else:
-        csi_producer = ingest.start_processing
-        is_live = True
+        csi_producer = ingest.RealtimeCSIProducer()

where_fi/collection/__init__.py

@@ -1,9 +1,24 @@
+from typing import Iterator
+
+import numpy as np
+import numpy.typing as npt
+
 from . import file, ingest
-from .protocols import CSICallback, CSIProducer
+from .protocols import CSIProducer, MergedCSI
 
 
-def noop(csi_callback: CSICallback | None = None) -> None:
-    del csi_callback
+class NoopCSIProducer:
+    """A no-op CSI producer that does nothing"""
+
+    is_live = False
+
+    def __call__(self) -> Iterator[MergedCSI]:
+        return iter([])
+
+    def stop(self) -> None:
+        pass
 
 
-__all__ = ["file", "ingest", "noop", "CSIProducer", "CSICallback"]
+CSIMatrix = npt.NDArray[np.complex64]
+
+__all__ = ["file", "ingest", "NoopCSIProducer", "CSIProducer"]

where_fi/collection/file.py

@@ -2,6 +2,7 @@ import logging
 import time
 from datetime import datetime, timedelta
 from pathlib import Path
+from typing import Iterator
 
 import h5py
 
@@ -10,30 +11,50 @@ from . import protocols
 logger = logging.getLogger(__name__)
 
 
-def start_processing(
-    file_path: Path,
-    csi_callback: protocols.CSICallback | None = None,
-) -> None:
-    logger.info(f"Replaying CSI data from {file_path}")
-    with h5py.File(file_path, "r") as file:
-        try:
-            for key in file:
-                datetime.fromisoformat(key)
-        except ValueError as e:
-            logger.exception(
-                "The file provided was not generated using this software", e
-            )
+class FileCSIPRoducer:
+    is_live = False
 
-        start_time = datetime.fromisoformat(list(file.keys())[0])
-        target_offset = datetime.now() - start_time
-        for key in file:
-            logger.debug(f"Sending data from {key} at {datetime.now().isoformat()}")
-            csi_callback(file[key][:])
-            curr_time_virtual = datetime.fromisoformat(key)
-            new_offset = datetime.now() - curr_time_virtual
-            logger.debug(f"New offset {new_offset}, target is {target_offset}")
-            if new_offset > target_offset + timedelta(seconds=1):
-                logger.warning(
-                    f"Data is {new_offset - target_offset} behind, lagging behind..."
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        if not self.path.exists():
+            raise FileNotFoundError(f"File {self.path} does not exist")
+
+    def __call__(self) -> Iterator[protocols.MergedCSI]:
+        """
+        Replay the CSI data from the file.
+        """
+        logger.info(f"Replaying CSI data from {self.path}")
+        with h5py.File(self.path, "r") as file:
+            try:
+                for key in file:
+                    datetime.fromisoformat(key)
+            except ValueError as e:
+                logger.exception(
+                    "The file provided was not generated using this software", e
                 )
-            time.sleep(max(0, (target_offset - new_offset).total_seconds()))
+
+            start_time = datetime.fromisoformat(list(file.keys())[0])
+            target_offset = datetime.now() - start_time
+            for key in file:
+                logger.debug(f"Sending data from {key} at {datetime.now().isoformat()}")
+
+                # TODO: add raw frames
+                yield protocols.MergedCSI(
+                    frames={},
+                    matrix=file[key][:],
+                )
+
+                curr_time_virtual = datetime.fromisoformat(key)
+                new_offset = datetime.now() - curr_time_virtual
+                logger.debug(f"New offset {new_offset}, target is {target_offset}")
+                if new_offset > target_offset + timedelta(seconds=1):
+                    logger.warning(
+                        f"Data is {new_offset - target_offset} behind, lagging behind..."
+                    )
+                time.sleep(max(0, (target_offset - new_offset).total_seconds()))
+
+    def stop(self) -> None:
+        """
+        Stop the producer.
+        """
+        logger.info("Stopping file CSI producer")

where_fi/collection/ingest.py

@@ -1,18 +1,18 @@
 import logging
 import multiprocessing as mp
+import selectors
 import socket
-import struct
 import subprocess
 import threading
 import time
 from datetime import datetime
-from typing import Callable, NamedTuple
+from typing import Iterator
 
 import numpy as np
 
 from ..config import config
 from .csi_frame import CSI
-from .protocols import CSICallback
+from .protocols import MergedCSI
 
 Host = tuple[str, int]
 
@@ -27,21 +27,30 @@ class FeitHost:
         self.active = True
         self.checker = threading.Thread(target=self.check_continuous)
         self.checker.start()
+        self.selectors: list[selectors.BaseSelector] = []
 
     def check_connection(self) -> bool:
-        feitcsi_status = subprocess.run(
-            f"ssh root@{self.host[0]} pgrep feitcsi",
-            check=False,
-            stdout=subprocess.DEVNULL,
-            shell=True,
-        )
-        return feitcsi_status.returncode == 0
+        try:
+            feitcsi_status = subprocess.run(
+                f"ssh root@{self.host[0]} pgrep feitcsi",
+                check=False,
+                stdout=subprocess.DEVNULL,
+                shell=True,
+                timeout=1,
+            )
+            return feitcsi_status.returncode == 0
+        except subprocess.TimeoutExpired:
+            return False
 
     def connect(self) -> None:
         self.server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         self.server.connect(self.host)
         self.server.send(b"stop\n")
         self.server.send(self.command.encode())
+
+        for selector in self.selectors:
+            selector.register(self.server, selectors.EVENT_READ, data=self)
+
         self.logger.info(f"Connected to {self.host}")
 
     def check_continuous(self) -> None:
@@ -54,6 +63,7 @@ class FeitHost:
         """
         last_status = False
         while self.active:
+            self.logger.debug(f"Checking connection to {self.host[0]}")
             if not self.check_connection():
                 self.logger.error(f"FeitCSI is not running on {self.host[0]}")
                 last_status = False
@@ -72,12 +82,17 @@ class FeitTransmitter(FeitHost):
             f"--channel-width {config.channel_width} "
             f"--format {config.frame_format} "
             f"--mode inject -s 1 --verbose "
-            f"--inject-delay {1_000_000 // config.sample_rate}"
+            f"--inject-delay {1_000_000 // config.collection_sample_rate}"
         )
         super().__init__(config.transmit_host, command)
 
 
 class FeitReceiver(FeitHost):
+    """
+    A class used to connect to the host running FeitCSI and receive CSI data over a UDP
+    socket.
+    """
+
     def __init__(self, host: Host) -> None:
         command = (
             f"feitcsi --frequency {config.central_freq} "
@@ -87,38 +102,51 @@ class FeitReceiver(FeitHost):
         )
         super().__init__(host, command)
 
-    def listen(self, queue: "mp.Queue[CSI]") -> None:
-        prev_time = datetime.now()
-        self.logger.info("Listening for CSI data")
-        while self.active:
-            while not hasattr(self, "server"):
-                time.sleep(0.1)
-            # This is the max size of a UDP packet. The size of the actual CSI
-            # packet will depend on the frame format and channel width, which
-            # changes the number of subcarriers
-            data = self.server.recv(65535)
-            try:
-                csidata = CSI(data)
-                self.logger.debug(
-                    f"Received CSI data after {datetime.now() - prev_time}"
-                )
-                prev_time = datetime.now()
-                queue.put(csidata)
-            except struct.error:
-                self.logger.error("Failed to parse CSI data")
-        self.logger.info("Stopping CSI receiver")
+    def recv(self) -> CSI:
+        """
+        Block until a CSI frame is received and return it.
+        """
+
+        # Receive the data from the socket
+        data = self.server.recv(65535)
+        # Decode the data using the CSI frame format
+        csi = CSI(data)
+        return csi
+
+    def register(self, selector: selectors.BaseSelector) -> None:
+        """
+        Register the receiver as a selector. This will allow us to multiplex the I/O
+        operations on a single thread.
+
+        This allows us to block until any of the receivers receive data
+        """
+        self.selectors.append(selector)
+
+        if hasattr(self, "server"):
+            selector.register(self.server, selectors.EVENT_READ, data=self)
 
 
-class CSIProcessor:
-    def __init__(
-        self,
-        receiver_connections: dict[Host, "mp.Queue[CSI]"],
-    ) -> None:
+class CSIAntennaArray:
+    """
+    Represents a set of receiver hosts that are used to receive CSI data, assumed to be
+    part of a linear antenna array.
+
+    This class is used to buffer a set of CSI data from multiple receivers, and once a
+    set of readings is ready for each receiver, it will be merged into a single sample
+    using the antenna order in the configuration.
+
+    When the data is ready, the following callbacks are called:
+        - `pre_merge_callback`: Called with the data before merging. This is useful for
+          per-antenna analysis, e.g. finding the correct antenna order
+        - `sample_callback`: Called with the merged data
+    """
+
+    def __init__(self, receivers: list[FeitReceiver]) -> None:
         self.pending_data: dict[Host, tuple[datetime, CSI]] = {}
         self.pending_data_lock = mp.Lock()
         self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
         self.last_processed = datetime.now()
-        self.connections = receiver_connections
+        self.receivers = receivers
         self.active = True
 
     def add_data(self, host: Host, data: CSI) -> None:
@@ -142,16 +170,13 @@ class CSIProcessor:
                 return False
         return True
 
-    def process_data(
-        self,
-        callback: CSICallback | None = None,
-        pre_merge_callback: Callable[[dict[Host, CSI]], None] | None = None,
-    ) -> None:
+    def process_data(self) -> None | MergedCSI:
+        if not self.is_ready():
+            self.logger.debug("Not all data is ready")
+            return None
         self.last_processed = datetime.now()
-        if pre_merge_callback is not None:
-            pre_merge_callback(
-                {host: data[1] for host, data in self.pending_data.items()}
-            )
+
+        frames = {host: data[1] for host, data in self.pending_data.items()}
         antenna_data = [
             np.expand_dims(self.pending_data[ip][1].matrix[:, antenna], axis=2)
             for ip, antenna in config.antennas.order
@@ -159,66 +184,45 @@ class CSIProcessor:
 
         # We have data from all servers
         all_data = np.concat(antenna_data, axis=1)
+        return MergedCSI(frames=frames, matrix=all_data)
 
-        if callback:
-            callback(all_data)
+    def process_forever(self) -> Iterator[MergedCSI]:
+        # Register the receivers with the selector
+        self.logger.info("Starting to process data from receivers")
+        selector = selectors.DefaultSelector()
 
-    def process_forever(
-        self,
-        callback: CSICallback | None = None,
-        pre_merge_callback: Callable[[dict[Host, CSI]], None] | None = None,
-    ) -> None:
-        try:
-            while self.active:
-                for ip, queue in self.connections.items():
-                    while not queue.empty():
-                        self.add_data(ip, queue.get())
-                if self.is_ready():
-                    self.process_data(callback, pre_merge_callback=pre_merge_callback)
-                else:
-                    self.logger.debug("Not all data is ready")
-                time.sleep(0.0005)
-        except KeyboardInterrupt:
-            self.logger.info("Exiting CSI processing")
+        for receiver in self.receivers:
+            receiver.register(selector)
+
+        while self.active:
+            events = selector.select(timeout=0.1)
+            if not events:
+                self.logger.warning("No events received in the last 0.1 seconds")
+                continue
+
+            for key, _ in events:
+                receiver = key.data
+                self.add_data(receiver.host, receiver.recv())
+
+            sample = self.process_data()
+            if sample is not None:
+                yield sample
 
 
-class Receiver(NamedTuple):
-    ip: Host
-    receiver: FeitReceiver
-    queue: "mp.Queue[CSI]"
+class RealtimeCSIProducer:
+    def __init__(self) -> None:
+        self.receivers = [FeitReceiver(ip) for ip in config.receive_hosts]
+        self.transmitter = FeitTransmitter()
 
+        # Start injecting CSI frames
+        self.buffer = CSIAntennaArray(self.receivers)
+        self.is_live = True
 
-def start_processing(
-    csi_callback: CSICallback | None = None,
-    pre_merge_callback: Callable[[dict[Host, CSI]], None] | None = None,
-) -> None:
-    receivers = [
-        Receiver(ip, FeitReceiver(ip), mp.Queue(config.sample_rate))
-        for ip in config.receive_hosts
-    ]
+    def __call__(self) -> Iterator[MergedCSI]:
+        return self.buffer.process_forever()
 
-    transmitter = FeitTransmitter()
-
-    # Start injecting CSI frames
-    processor = CSIProcessor({r.ip: r.queue for r in receivers})
-
-    receiver_processes = [
-        threading.Thread(target=r.receiver.listen, args=(r.queue,)) for r in receivers
-    ]
-
-    for proc in receiver_processes:
-        proc.start()
-
-    processing_thread = mp.Process(
-        target=processor.process_forever, args=(csi_callback, pre_merge_callback)
-    )
-    processing_thread.start()
-    try:
-        while True:
-            time.sleep(100)
-    except KeyboardInterrupt:
-        for r in receivers:
-            r.receiver.active = False
-        transmitter.active = False
-        processor.active = False
-        return
+    def stop(self) -> None:
+        for r in self.receivers:
+            r.active = False
+        self.transmitter.active = False
+        self.buffer.active = False

where_fi/collection/protocols.py

@@ -1,10 +1,22 @@
-from typing import Callable, Protocol
+from typing import Iterator, NamedTuple, Protocol
 
 import numpy as np
 import numpy.typing as npt
 
-CSICallback = Callable[[npt.NDArray[np.complex64]], None]
+from .csi_frame import CSI
+
+CSIHost = tuple[str, int]
+
+
+class MergedCSI(NamedTuple):
+    """Merged CSI data from all antennas"""
+
+    frames: dict[CSIHost, CSI]
+    matrix: npt.NDArray[np.complex64]
 
 
 class CSIProducer(Protocol):
-    def __call__(self, csi_callback: CSICallback | None = None) -> None: ...
+    is_live: bool
+
+    def __call__(self) -> Iterator[MergedCSI]: ...
+    def stop(self) -> None: ...

where_fi/config/models.py

@@ -43,7 +43,8 @@ class Config(BaseModel):
 
     antennas: Antennas
 
-    sample_rate: int
+    collection_sample_rate: int
+    processing_sample_rate: int
     central_freq: int
     channel_width: Literal[20, 40, 80, 160]
     frame_format: Literal["NOHT", "HT", "VHT", "HESU"]

where_fi/processing/preprocess.py

@@ -6,44 +6,107 @@ import numpy as np
 import numpy.typing as npt
 from scipy.signal import butter, correlate, sosfilt, sosfilt_zi
 
+from where_fi.collection import CSIMatrix
+
 from ..config import config
 from ..visualise import server as visualise
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
 
 np.seterr(invalid="ignore")
 
 
 class Preprocessor:
     def __init__(self) -> None:
-        self.prev_entries: Queue[npt.NDArray[np.complex64]] = Queue(maxsize=100)
         self.short_term_avg = np.zeros((1,), dtype=np.complex64)
         self.long_term_avg = np.zeros((1,), dtype=np.complex64)
-        self.filter = butter(
-            5,
-            config.preprocessing.bandpass.bounds,
-            fs=config.sample_rate,
-            btype="band",
-            output="sos",
-        )
+        # self.filter = butter(
+        #     5,
+        #     config.preprocessing.bandpass.bounds,
+        #     fs=config.sample_rate,
+        #     btype="band",
+        #     output="sos",
+        # )
+        self._last_sample = None
+
+    @property
+    def last_sample(self) -> None | CSIMatrix:
+        """
+        The last sample of the preprocessor. This is used for low frequency processing
+        """
+        return self._last_sample
+
+    def remove_sto(self, csi: CSIMatrix) -> CSIMatrix:
+        """
+        Remove sampling time offsets caused by:
+            - Sampling frequency offset
+            - Packet detection delay
+
+        This is done by multiplying the CSI matrices of consecutive antennas in the
+        array.
+
+        According to [1]:
+            > Conjugate multiplication and division are the only two methods to
+            > eliminate the SFO and PDD.
+
+        No citation or explanation is provided, so not sure why/whether it works.
+        Something similar is also done in [2] without explanation.
+
+        [1] - https://tns.thss.tsinghua.edu.cn/wst/docs/sanitization
+        [2] - https://doi.org/10.1109/ICC51166.2024.10623053
+        """
+
+        csi_remove_sto = np.zeros_like(csi)
+        for antenna in range(csi.shape[1]):
+            antenna_nxt = (antenna + 1) % csi.shape[1]
+            csi_remove_sto[:, antenna, :] = np.multiply(
+                csi[:, antenna, :], csi[:, antenna_nxt, :].conj()
+            )
+        return csi_remove_sto
+
+    def remove_sfo(
+        self,
+        csi: CSIMatrix,
+        visualiser: None
+        | Callable[[npt.NDArray[Any], visualise.figures.Figure], None] = None,
+    ) -> CSIMatrix:
+        """
+        Remove sampling frequency offsets by linear regression.
+
+        This is caused by the difference in sampling frequency between the transmitter
+        and receiver, and this is linear in frequency. We can estimate it using linear
+        regression and compensate for it.
+        """
+        N_st, N_rx, _ = csi.shape
+        unwrapped = np.unwrap(np.angle(csi[:, :, 0]), axis=0).reshape(N_st, N_rx, 1)
+
+        if visualiser:
+            visualiser(unwrapped, visualise.figures.Figure.UNWRAPPED_PHASE)
+
+        X = np.vstack([np.arange(N_st), np.ones(N_st)]).T
+        for antenna in range(csi.shape[1]):
+            tau, rho = np.linalg.lstsq(X, unwrapped[:, antenna, 0])[0]
+            csi[:, antenna, 0] = np.abs(csi[:, antenna, 0]) * np.exp(
+                1j
+                * (np.angle(csi[:, antenna, 0]) - (tau * np.arange(csi.shape[0]) + rho))
+            )
+
+        return csi
 
     def preprocess(
         self,
-        h: npt.NDArray[np.complex64],
+        h: CSIMatrix,
         visualiser: None
-        | Callable[[npt.NDArray[Any], visualise.DataType], None] = None,
-    ) -> npt.NDArray[np.complex64]:
+        | Callable[[npt.NDArray[Any], visualise.figures.Figure], None] = None,
+    ) -> CSIMatrix:
         # CSI data is not available for pilot subcarriers.
-        h_hat: npt.NDArray[np.complex64] = np.where(
+        h_hat: CSIMatrix = np.where(
             np.expand_dims(h[:, 0, 0] == 0, axis=(1, 2)),
             correlate(h, [[[1 / 2]], [[0]], [[1 / 2]]], mode="same"),
             h,
         )
 
-        # Skip subcarrierss per config
-        h_hat = h_hat[:: config.preprocessing.subcarrier_step, :, :]
-        # logger.info(f"CSI shape: {h_hat.shape}")
+        logger.debug(f"CSI shape: {h_hat.shape}")
 
         # h_hat = np.multiply(h_hat, h_hat.conj() / abs(h_hat.conj()))
         h_hat = np.nan_to_num(h_hat)
@@ -51,27 +114,13 @@ class Preprocessor:
         # h_hat = correlate(h_hat, np.ones((3, 1, 1)) / 3)
         # h_hat = correlate(h_hat, [[[1 / 4]], [[1 / 2]], [[1 / 4]]], mode="valid")
 
-        # Unwrap phase and remove linear fit
-        print(h_hat.shape)
-        unwrapped = np.unwrap(np.angle(h_hat[:, :, 0]), axis=0).reshape(
-            h_hat.shape[0], h_hat.shape[1], 1
-        )
-        if visualiser:
-            visualiser(unwrapped, visualise.DataType.UNWRAPPED_PHASE)
-        for antenna in range(h_hat.shape[1]):
-            tau, rho = np.linalg.lstsq(
-                np.vstack([np.arange(h_hat.shape[0]), np.ones(h_hat.shape[0])]).T,
-                unwrapped[:, antenna, 0],
-            )[0]
-            h_hat[:, antenna, 0] = np.abs(h_hat[:, antenna, 0]) * np.exp(
-                1j
-                * (
-                    np.angle(h_hat[:, antenna, 0])
-                    - (tau * np.arange(h_hat.shape[0]) + rho)
-                )
-            )
+        h_hat = self.remove_sfo(h_hat, visualiser=visualiser)
+
+        # Skip subcarrierss per config
+        h_hat = h_hat[:: config.preprocessing.subcarrier_step, :, :]
+
+        # h_hat = self.remove_sto(h_hat)
 
-        return h_hat
         # Assume that all csi matrices will have the same shape
         if self.long_term_avg.shape != h_hat.shape:
             self.long_term_avg = np.zeros(h_hat.shape, dtype=np.complex64)
@@ -81,7 +130,10 @@ class Preprocessor:
             + h_hat * config.preprocessing.moving_average_alpha
         )
 
+        self._last_sample = h_hat
+
         # Remove long term average, to remove static paths
+        return h_hat
         h_hat -= self.long_term_avg
 
         # Apply bandpass filter to remove low and high frequency noise

where_fi/utils/antenna_order.py

@@ -2,6 +2,8 @@ import logging
 from collections import deque
 from typing import Any, Collection
 
+from where_fi.application import CSIApplication
+
 from ..collection import ingest
 from ..config import config
 
@@ -11,11 +13,19 @@ AntennaIdentifier = tuple[ingest.Host, int]
 order: list[AntennaIdentifier] = []
 prev_unplugged: set[AntennaIdentifier] = set()
 long_antenna_average: dict[AntennaIdentifier, deque[int]] = {
-    (host, 0): deque(maxlen=15 * config.sample_rate) for host in config.receive_hosts
-} | {(host, 1): deque(maxlen=15 * config.sample_rate) for host in config.receive_hosts}
+    (host, 0): deque(maxlen=15 * config.collection_sample_rate)
+    for host in config.receive_hosts
+} | {
+    (host, 1): deque(maxlen=15 * config.collection_sample_rate)
+    for host in config.receive_hosts
+}
 short_antenna_average: dict[AntennaIdentifier, deque[int]] = {
-    (host, 0): deque(maxlen=2 * config.sample_rate) for host in config.receive_hosts
-} | {(host, 1): deque(maxlen=2 * config.sample_rate) for host in config.receive_hosts}
+    (host, 0): deque(maxlen=2 * config.collection_sample_rate)
+    for host in config.receive_hosts
+} | {
+    (host, 1): deque(maxlen=2 * config.collection_sample_rate)
+    for host in config.receive_hosts
+}
 
 RSSI_THRESHOLD = 10
 
@@ -24,6 +34,10 @@ def average(data: Collection[Any]) -> float:
     return sum(data) / len(data)
 
 
+app = CSIApplication(ingest.RealtimeCSIProducer())
+
+
+@app.on_pre_merge
 def callback(antenna_data: dict[ingest.Host, ingest.CSI]) -> None:
     global prev_unplugged
     global antenna_average
@@ -65,4 +79,4 @@ def callback(antenna_data: dict[ingest.Host, ingest.CSI]) -> None:
 
 
 def main() -> None:
-    ingest.start_processing(pre_merge_callback=callback)
+    app.start()

where_fi/visualise/.gitignore

@@ -1,2 +1,2 @@
-server/generated
+generated
 frontend/src/grpc

where_fi/visualise/Makefile

@@ -1,9 +1,10 @@
-all: server/generated frontend/src/grpc
+all: generated frontend/src/grpc
 
-server/generated: protos/*.proto
-	mkdir -p server/generated
-	find protos/ -type f -name "*.proto" | xargs uv run python -m grpc_tools.protoc -Iprotos --python_out=server/generated --pyi_out=server/generated --grpc_python_out=server/generated
-	find protos/ -type f -name "*.proto" | xargs uv run protol --create-package --in-place --python-out server/generated/  protoc --proto-path=protos/
+generated: protos/*.proto
+	mkdir -p generated
+	find protos/ -type f -name "*.proto" | xargs uv run python -m grpc_tools.protoc -Iprotos --python_out=generated --pyi_out=generated --grpc_python_out=generated --mypy_grpc_out=generated
+
+	find protos/ -type f -name "*.proto" | xargs uv run protol --create-package --in-place --python-out generated/  protoc --proto-path=protos/
 
 frontend/src/grpc: protos/*.proto
 	mkdir -p frontend/src/grpc

where_fi/visualise/extract/__init__.py

@@ -0,0 +1,20 @@
+import logging
+
+import grpc
+
+from ..generated import figure_pb2, figure_pb2_grpc
+
+
+def run() -> None:
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel("localhost:50051") as channel:
+        stub = figure_pb2_grpc.FigureServiceStub(channel)
+        for figure in stub.GetFigure(figure_pb2.FigureRequest()):
+            print(f"Figure: {figure}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig()
+    run()

where_fi/visualise/extract/__main__.py

@@ -0,0 +1,89 @@
+from pathlib import Path
+from typing import cast
+
+import grpc
+import matplotlib.pyplot as plt
+import numpy as np
+import typer
+
+from ..generated import figure_pb2, figure_pb2_grpc
+
+app = typer.Typer()
+
+
+@app.command()
+def list() -> None:
+    """
+    List all figures from the gRPC service.
+    """
+    print("Connecting to the server...")
+    with grpc.insecure_channel("localhost:50051") as channel:
+        stub = figure_pb2_grpc.FigureServiceStub(channel)
+        print("Retrieving available figures...\n")
+        figures = [x for x in stub.GetFigure(figure_pb2.FigureRequest())]
+
+        if not figures:
+            print("No figures found.")
+        else:
+            for i, figure in enumerate(figures, 1):
+                print(f"{i}. Figure: {figure.title} (ID: {figure.uuid})")
+
+            choice = typer.prompt("Which figure to extract?", type=int)
+            assert isinstance(choice, int)
+            if 1 <= choice <= len(figures):
+                selected_figure = figures[choice - 1]
+                print(f"You selected: {selected_figure.title}")
+                extract(selected_figure.uuid)
+            else:
+                print("Invalid selection. Exiting.")
+
+
+def get_figure(uuid: str) -> tuple[figure_pb2.Figure, figure_pb2.FigureData]:
+    """
+    Get the figure data for a specific UUID.
+    """
+    print(f"Connecting to the server to extract data for UUID: {uuid}...")
+    with grpc.insecure_channel("localhost:50051") as channel:
+        stub = figure_pb2_grpc.FigureServiceStub(channel)
+        all_figures = {x.uuid: x for x in stub.GetFigure(figure_pb2.FigureRequest())}
+        figure_data = stub.GetFigureUpdate(figure_pb2.FigureDataRequest(uuid=uuid))
+        for data in figure_data:
+            return (all_figures[uuid], data)
+    raise ValueError(f"Figure with UUID {uuid} not found.")
+
+
+@app.command()
+def extract(uuid: str, output: Path | None = None) -> None:
+    """
+    Extract data for a specific figure identified by its UUID.
+    """
+    _, figure = get_figure(uuid)
+    if not output:
+        output = cast(Path, typer.prompt("Enter output file path:", type=Path))
+    with open(output, "wb") as f:
+        f.write(figure.SerializeToString())
+
+
+@app.command()
+def plot(uuid: str) -> None:
+    """
+    Plot the figure data for a specific UUID.
+    """
+    figure, data = get_figure(uuid)
+    if data.line:
+        for line in data.line.lines:
+            plt.plot(
+                line.x if line.x else np.arange(len(line.y)), line.y, label=line.label
+            )
+        plt.xlabel(figure.x_label)
+        plt.ylabel(figure.y_label)
+        plt.title(figure.title)
+    elif figure.heatmap:
+        plt.imshow(figure.heatmap.data, cmap="hot", interpolation="nearest")
+    elif figure.histogram:
+        plt.hist(figure.histogram.data, bins=figure.histogram.bins)
+    plt.show()
+
+
+if __name__ == "__main__":
+    app()

where_fi/visualise/server/__init__.py

@@ -2,6 +2,7 @@ import logging
 import multiprocessing as mp
 import queue
 import threading
+import time
 from concurrent import futures
 from dataclasses import dataclass
 from typing import Any, Generator
@@ -10,13 +11,8 @@ import grpc
 import numpy as np
 import numpy.typing as npt
 
+from ..generated import figure_pb2, figure_pb2_grpc
 from . import figures
-from .generated import figure_pb2, figure_pb2_grpc
-
-logger = logging.getLogger(__name__)
-
-clients: dict[str, list[queue.Queue[figure_pb2.FigureData]]] = {}
-clients_lock = threading.Lock()
 
 
 @dataclass
@@ -26,25 +22,30 @@ class VisualiserData:
 
 
 class FigureServer(figure_pb2_grpc.FigureServiceServicer):
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(__name__)
+        self.clients: dict[str, list[queue.Queue[figure_pb2.FigureData]]] = {}
+        self.clients_lock = threading.Lock()
+
     def GetFigure(
         self, request: figure_pb2.FigureRequest, context: grpc.ServicerContext
     ) -> Generator[figure_pb2.Figure, None, None]:
         for figure_group in figures.Figure:
-            for figure in figures.all_figures[figure_group].figures:
+            for figure in figure_group.figure_class().figures:
                 yield figure
 
     def GetFigureUpdate(
         self, request: figure_pb2.FigureDataRequest, context: grpc.ServicerContext
     ) -> Generator[figure_pb2.FigureData, None, None]:
-        logger.info(
+        self.logger.info(
             f"Received request for figure data stream for figure {request.uuid}"
         )
 
-        with clients_lock:
+        with self.clients_lock:
             q: queue.Queue[figure_pb2.FigureData] = queue.Queue()
-            if request.uuid not in clients:
-                clients[request.uuid] = []
-            clients[request.uuid].append(q)
+            if request.uuid not in self.clients:
+                self.clients[request.uuid] = []
+            self.clients[request.uuid].append(q)
 
         try:
             while context.is_active():
@@ -53,33 +54,52 @@ class FigureServer(figure_pb2_grpc.FigureServiceServicer):
                 except queue.Empty:
                     pass
         finally:
-            with clients_lock:
-                clients[request.uuid].remove(q)
+            with self.clients_lock:
+                self.clients[request.uuid].remove(q)
 
 
-def add_data(dtype: figures.Figure, new_data: npt.NDArray[np.complex128]) -> None:
-    updates = figures.all_figures[dtype].update(new_data)
-    for fig_id, update in updates.items():
-        for client in clients.get(fig_id, []):
-            client.put(update)
+class Webapp:
+    def __init__(self) -> None:
+        self.logger = logging.getLogger(__name__)
+        self.active = True
+        self.figure_server = FigureServer()
 
+    def add_data(
+        self, dtype: figures.Figure, new_data: npt.NDArray[np.complex128]
+    ) -> None:
+        updates = figures.all_figures[dtype].update(new_data)
+        for fig_id, update in updates.items():
+            for client in self.figure_server.clients.get(fig_id, []):
+                client.put(update)
 
-def listen_for_data(data_queue: "mp.Queue[VisualiserData]") -> None:
-    while True:
-        data = data_queue.get()
-        add_data(data.dtype, data.data)
+    def listen_for_data(self, data_queue: "mp.Queue[VisualiserData]") -> None:
+        while self.active:
+            self.logger.debug("Listening for data")
+            try:
+                data = data_queue.get(timeout=0.5)
+                self.add_data(data.dtype, data.data)
+            except queue.Empty:
+                pass
 
+        while not data_queue.empty():
+            data = data_queue.get()
 
-def start(data_queue: "mp.Queue[VisualiserData]") -> None:
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-    figure_pb2_grpc.add_FigureServiceServicer_to_server(FigureServer(), server)
-    server.add_insecure_port("[::]:50051")
-    server.add_insecure_port("0.0.0.0:50051")
-    logger.info("Starting server on port 50051")
-    server.start()
-    logger.info("Server started")
-    threading.Thread(target=listen_for_data, args=(data_queue,), daemon=True).start()
-    try:
-        server.wait_for_termination()
-    except KeyboardInterrupt:
-        logger.info("Exiting visualisation server")
+    def start(self, data_queue: "mp.Queue[VisualiserData]") -> None:
+        grpc_server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        figure_pb2_grpc.add_FigureServiceServicer_to_server(
+            self.figure_server, grpc_server
+        )
+        grpc_server.add_insecure_port("[::]:50051")
+        grpc_server.add_insecure_port("0.0.0.0:50051")
+        self.logger.info("Starting server on port 50051")
+        grpc_server.start()
+        self.logger.info("Server started")
+
+        data_thread = threading.Thread(target=self.listen_for_data, args=(data_queue,))
+        data_thread.start()
+
+        while self.active:
+            time.sleep(1)
+            self.logger.debug("Server is running")
+        grpc_server.stop(0.5)
+        data_thread.join()

where_fi/visualise/server/figures.py

@@ -58,10 +58,17 @@ class SimpleLineChart:
         )
 
     def update(
-        self, new_data: npt.NDArray[np.complex128], labels: Sequence[str]
+        self,
+        new_data: npt.NDArray[np.complex128],
+        labels: Sequence[str],
+        x_values: None | npt.NDArray[np.float64] = None,
     ) -> FigureUpdate:
         lines = [
-            line_pb2.LineChartData.Line(y=new_data[i], label=labels[i])
+            line_pb2.LineChartData.Line(
+                y=new_data[i],
+                label=labels[i],
+                x=x_values[i] if x_values is not None else None,
+            )
             for i in range(new_data.shape[0])
         ]
         return {
@@ -72,6 +79,30 @@ class SimpleLineChart:
         }
 
 
+class PerSubcarrierFigure(SpecificFigure):
+    def __init__(
+        self,
+        figures: Sequence[SimpleLineChart],
+        funcs: Sequence[Callable[[npt.NDArray[Any]], npt.NDArray[Any]]],
+    ) -> None:
+        self.charts = figures
+        self.figures = [figure.figure for figure in figures]
+        self.funcs = funcs
+
+    def update(self, new_data: npt.NDArray[np.complex128]) -> FigureUpdate:
+        """
+        Update the figure with new data.
+
+        The data is expected to be in the shape (subcarriers, data).
+        """
+        subcarrier_labels = [f"Subcarrier {i}" for i in range(new_data.shape[0])]
+        updates = [
+            figure.update(func(new_data), labels=subcarrier_labels)
+            for func, figure in zip(self.funcs, self.charts, strict=True)
+        ]
+        return reduce((lambda a, b: a | b), updates)
+
+
 class PerAntennaFigure(SpecificFigure):
     """A figure that plots data for each antenna separately.
 
@@ -156,12 +187,90 @@ class HeatmapFigure(SpecificFigure):
         return {self.figure.uuid: figure_pb2.FigureData(heatmap=heatmap)}
 
 
+class EmpiricalCDF(SpecificFigure):
+    """Helper class to create a graph of the empirical cumulative distribution function
+    and probability density functions of a set of observations.
+    """
+
+    def __init__(self, title: str, x_label: str) -> None:
+        self.chart = SimpleLineChart(title, x_label, "Cumulative Probability")
+        self.figures = [self.chart.figure]
+
+    def update(self, new_data: npt.NDArray[np.float64]) -> FigureUpdate:
+        new_data = np.sort(new_data, axis=-1)
+        if new_data.ndim == 1:
+            new_data = np.expand_dims(new_data, 0)
+        y_values = np.repeat(
+            np.linspace(0, 1, new_data.shape[-1])[np.newaxis, :],
+            new_data.shape[0],
+            axis=0,
+        )
+        return self.chart.update(y_values, labels=["Frequency"], x_values=new_data)
+
+
+class RandomVariable(SpecificFigure):
+    def get_cdf(
+        self, new_data: npt.NDArray[np.float64]
+    ) -> tuple[npt.NDArray[np.floating[Any]], npt.NDArray[np.floating[Any]]]:
+        """Get the cumulative distribution function of the data.
+
+        The data is expected to be in the shape (lines, data) or (data,) for single-line
+        charts.
+        """
+        new_data = np.sort(new_data, axis=-1)
+        if new_data.ndim == 1:
+            new_data = np.expand_dims(new_data, 0)
+            y_values = np.expand_dims(np.linspace(0, 1, new_data.size), 0)
+        else:
+            y_values = np.repeat(
+                np.linspace(0, 1, new_data.shape[-1])[np.newaxis, :],
+                new_data.shape[0],
+                axis=0,
+            )
+        return new_data, y_values
+
+    def __init__(self, x_label: str, line_type: str = "Subcarrier") -> None:
+        self.charts = [
+            # SimpleLineChart(
+            #     f"{x_label} Probability Density", x_label, "Probability Density"
+            # ),
+            SimpleLineChart(
+                f"{x_label} Cumulative Distribution", x_label, "Cumulative Probability"
+            ),
+        ]
+        self.funcs = [
+            # self.get_pdf,
+            self.get_cdf
+        ]
+        self.figures = [figure.figure for figure in self.charts]
+
+    def update(self, new_data: npt.NDArray[np.complex128]) -> FigureUpdate:
+        """
+        Update the figure with new data.
+
+        The data is expected to be in the shape (subcarriers, data).
+        """
+        subcarrier_labels = [f"Subcarrier {i}" for i in range(new_data.shape[0])]
+        updates: list[FigureUpdate] = []
+        for func, figure in zip(self.funcs, self.charts, strict=True):
+            x, y = func(new_data)
+            updates.append(
+                figure.update(
+                    y,
+                    labels=subcarrier_labels,
+                    x_values=x,
+                )
+            )
+        return reduce((lambda a, b: a | b), updates)
+
+
 class Figure(Enum):
     RAW_CSI = 0
     UNWRAPPED_PHASE = 1
     PROCESSED_CSI = 2
     MUSIC_EIGENVALUES = 3
     AOA_HEATMAP = 4
+    PHASE_ANALYSIS = 5
 
     def figure_class(self) -> SpecificFigure:
         return all_figures[self]
@@ -187,4 +296,5 @@ all_figures = {
     ),
     Figure.MUSIC_EIGENVALUES: MusicEigenvalueHistogram(),
     Figure.AOA_HEATMAP: HeatmapFigure(),
+    Figure.PHASE_ANALYSIS: RandomVariable("Phase"),
 }