init

lib_audio_dsp/python/audio_dsp/design/__init__.py

@@ -0,0 +1,3 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""Classes required for constructing the DSP pipeline."""

lib_audio_dsp/python/audio_dsp/design/_draw.py

@@ -0,0 +1,21 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""some graphviz helpers."""
+
+import graphviz  # type: ignore
+
+
+def new_record_digraph():
+    """
+    Create a digraph with some attributes set.
+
+    Returns
+    -------
+    graphviz.Digraph
+        A graph object
+    """
+    dot = graphviz.Digraph()
+    dot.clear()
+    dot.attr(ranksep="1.0")
+    dot.attr("node", shape="record")
+    return dot

lib_audio_dsp/python/audio_dsp/design/build_utils.py

@@ -0,0 +1,332 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""Utility functions for building and running the application within the Jupyter notebook."""
+
+import IPython
+import ipywidgets as widgets
+import pathlib
+import shlex
+import subprocess
+import time
+
+NullablePathLike = str | pathlib.Path | None
+
+
+class XCommonCMakeHelper:
+    """
+    This class packages a set of helper utilities for configuring, building, and
+    running xcore applications using xcommon-cmake within Python.
+
+    Parameters
+    ----------
+    source_dir : str | pathlib.Path | None
+        Specify a source directory for this build, passed as the -S parameter to
+        CMake. If None passed or unspecified, defaults to the current working
+        directory.
+    build_dir : str | pathlib.Path | None
+        Specify a build directory for this build, passed as the -B parameter to
+        CMake. If None passed or unspecified, defaults to "build" within the
+        current working directory.
+    bin_dir : str | pathlib.Path | None
+        Specify a binary output directory for this build. This should match what
+        is configured to be the output directory from "cmake --build" within the
+        application. If None passed or unspecified, defaults to "bin" within the
+        current working directory.
+    project_name : str | None
+        The name of the project() specified in the project's CMakeLists.txt.
+        If None or unspecified, defaults to the name of the current working
+        directory (so if in /app_example_name/, the project name is assumed to
+        be app_example_name).
+    config_name : str | None
+        The name of the configuration to use from the project's CMakeLists.txt.
+        If None or unspecified, defaults to nothing - therefore the --target
+        option to CMake will be just the project name, and the output binary
+        will be assumed to be
+        "<current working directory>/<bin_dir>/<project_name>.xe". If specified,
+        the --target option to CMake will be "<project name>_<config name>", and
+        the output binary will be assumed to be
+        "<current working directory>/<bin_dir>/<config_name>/<project name>_<config name>.xe".
+
+    """
+
+    log_str = """<pre style="font-family: monospace, monospace;">{output}</pre>"""
+
+    def __init__(
+        self,
+        source_dir: NullablePathLike = None,
+        build_dir: NullablePathLike = None,
+        bin_dir: NullablePathLike = None,
+        project_name: str | None = None,
+        config_name: str | None = None,
+    ) -> None:
+        if source_dir is None:
+            self.source_dir = self._determine_source_dir()
+        else:
+            try:
+                self.source_dir = pathlib.Path(source_dir)
+            except TypeError as e:
+                raise TypeError("source_dir must be pathlike or None!") from e
+
+        if build_dir is None:
+            self.build_dir = self._determine_build_dir()
+        else:
+            try:
+                self.build_dir = pathlib.Path(build_dir)
+            except TypeError as e:
+                raise TypeError("build_dir must be pathlike or None!") from e
+
+        if bin_dir is None:
+            self.bin_dir = self._determine_bin_dir()
+        else:
+            try:
+                self.bin_dir = pathlib.Path(bin_dir)
+            except TypeError as e:
+                raise TypeError("bin_dir must be pathlike or None!") from e
+
+        if project_name is None:
+            self.project_name = self._determine_project_name()
+        elif isinstance(project_name, str):
+            self.project_name = project_name
+        else:
+            raise TypeError("project_name parameter must be str or None!")
+
+        if config_name is None:
+            self.config_suffix = self._determine_config_suffix()
+            self.config_name = self._determine_config_name()
+        elif isinstance(config_name, str):
+            self.config_suffix = "_" + config_name
+            self.config_name = config_name
+        else:
+            raise TypeError("config_name parameter must be str or None!")
+
+        self.target_name = self._determine_target_name()
+        self.configure_done: bool = False
+
+    def _determine_source_dir(self) -> pathlib.Path:
+        # We assume here that the CWD is the application directory, and that
+        #     this is the desired source directory.
+        return pathlib.Path.cwd()
+
+    def _determine_build_dir(self) -> pathlib.Path:
+        # We assume here that the CWD is the application directory, and that
+        #     this will contain a subdirectory /build/ which is the desired
+        #     build directory.
+        return pathlib.Path.cwd() / "build"
+
+    def _determine_bin_dir(self) -> pathlib.Path:
+        # We assume here that the CWD is the application directory, and that
+        #     this will contain a subdirectory /bin/ which is the bin
+        #     directory.
+        return pathlib.Path.cwd() / "bin"
+
+    def _determine_project_name(self) -> str:
+        # We assume here that the name of the project is the same as the
+        #     name of the enclosing directory.
+        return pathlib.Path.cwd().name
+
+    def _determine_config_suffix(self) -> str:
+        # We assume here that if no config has been specified then the default
+        #     target name is just the project name with no config suffix.
+        return ""
+
+    def _determine_config_name(self) -> str:
+        # We assume here that if no config has been specified then the default
+        #     config name is blank.
+        return ""
+
+    def _determine_target_name(self) -> str:
+        # We assume here that the default target name is always the project name
+        #     with the config suffix appended.
+        return self.project_name + self.config_suffix
+
+    def _log(self, process: subprocess.Popen, title: str = "") -> None:
+        widget = widgets.HTML(value="")
+        accordion = widgets.Accordion(children=[widget])
+        accordion.set_title(0, title)
+        IPython.display.display(accordion)  # pyright: ignore [reportAttributeAccessIssue]
+        output = ""
+        for line in process.stdout:  # pyright: ignore [reportOptionalIterable]
+            output += line
+            widget.value = XCommonCMakeHelper.log_str.format(output=output)
+        process.wait()
+        if process.returncode:
+            accordion.set_title(0, title + "  Failed ❌ (click for details)")
+        else:
+            accordion.set_title(0, title + "  ✔")
+
+    def _log_poll(self, process, title=""):
+        """
+        Log the run status of a process that isn't expected to finish, for example, xrun --xscope-port.
+        If the process does finish, log the stdout and indicate fail, otherwise indicate pass.
+        """
+        widget = widgets.HTML(value="")
+        accordion = widgets.Accordion(children=[widget])
+        accordion.set_title(0, title)
+        IPython.display.display(accordion)
+        time.sleep(4)
+        if process.poll():  # Process has ended when we don't expect it to
+            output = ""
+            for line in process.stdout:
+                output += line
+                widget.value = XCommonCMakeHelper.log_str.format(output=output)
+            accordion.set_title(0, title + "  Failed ❌ (click for details)")
+        else:
+            accordion.set_title(0, title + "  ✔")
+
+    def configure(self) -> int | None:
+        """
+        Invoke CMake with the options specified in this class instance.
+        Invokation will be of the form
+        ``cmake -S <source_dir> -B <build_dir>``. On first run, the invokation
+        will also contain ``-G <generator>``, where "generator"
+        will be either "Ninja" if Ninja is present on the current system or
+        "Unix Makefiles" if it is not.
+
+        Returns
+        -------
+        returncode
+            Return code from the invokation of CMake. 0 if success.
+        """
+        cache = self.build_dir / "CMakeCache.txt"
+        makefile = self.build_dir / "Makefile"
+        ninjabuild = self.build_dir / "build.ninja"
+        if (
+            (not self.configure_done)
+            or (not cache.exists())
+            or not (makefile.exists() or ninjabuild.exists())
+        ):
+            cmake_cmd = [
+                "cmake",
+                "-S",
+                f"{self.source_dir}",
+                "-B",
+                f"{self.build_dir}",
+                "-DCMAKE_COLOR_MAKEFILE=OFF",
+            ]
+            if cache.exists():
+                # Generator is already known by CMake
+                ret = subprocess.Popen(
+                    cmake_cmd,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                )
+            else:
+                # Use Makefiles as its the only one xcommon cmake officially supports
+                generator = "Unix Makefiles"
+                ret = subprocess.Popen(
+                    [
+                        *cmake_cmd,
+                        "-G",
+                        generator,
+                    ],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                )
+            self._log(ret, "Configuring...")
+            if not ret.returncode:
+                self.configure_done = True
+            return ret.returncode
+
+    def build(self) -> int:
+        """
+        Invoke CMake's build with the options specified in this class instance.
+        Invokation will be of the form
+        ``cmake --build <build_dir> --target <target_name>``, where the target
+        name is constructed as per this class' docstring.
+
+        Returns
+        -------
+        returncode
+            Return code from the invokation of CMake. 0 if success.
+        """
+        ret = subprocess.Popen(
+            ["cmake", "--build", f"{self.build_dir}", "--target", f"{self.target_name}"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+        self._log(ret, "Compiling...")
+        return ret.returncode
+
+    def run(self, xscope: bool = True, hostname: str = "localhost", port: str = "12345") -> int:
+        """
+        Invoke xrun or xgdb with the options specified in this class instance.
+
+        If xscope is True the invocation will be of the form::
+
+            xgdb -q --return-child-result --batch
+            -ex "connect --xscope-port <hostname>:<port> --xscope"
+            -ex "load"
+            -ex "continue"
+            <binary>
+
+        whereas if xscope if False the invocation will be of the form::
+
+            xrun <binary>
+
+        where the path to the binary is constructed as per this
+        class' docstring.
+
+        Parameters
+        ----------
+        xscope : bool
+            Specify whether to set up an xscope server or not.
+
+        hostname : str
+            Hostname to use for the xscope server if xscope is True
+
+        port : str
+            Port to use for the xscope server if xscope is True
+
+        Returns
+        -------
+        returncode
+            Return code from the invokation of xrun or xgdb. 0 if success.
+        """
+        app = (
+            f"{self.bin_dir / self.config_name / (self.project_name + self.config_suffix + '.xe')}"
+        )
+        cmd = ""
+        if xscope:
+            cmd += f'xgdb -q --return-child-result --batch -ex "connect --xscope-port {hostname}:{port} --xscope" -ex "load" -ex "continue"'
+        else:
+            cmd += "xrun"
+        cmd += f" {shlex.quote(app)}"
+        ret = subprocess.Popen(
+            shlex.split(cmd),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+        if xscope:
+            self._log_poll(ret, f"Running...")
+        else:
+            self._log(ret, f"Running...")
+        return ret.returncode
+
+    def configure_build_run(self, xscope: bool = True) -> None:
+        """
+        Run, in order, this class' .configure(), .build(), and .run() methods.
+        If any return code from any of the three is nonzero, returns early.
+        Otherwise, sleeps for 5 seconds after the .run() stage and prints
+        "Done!".
+
+        Parameters
+        ----------
+        xscope : bool
+            Passed directly to the call to .run(); determines whether to start
+            an xscope server or not.
+        """
+        returncode = self.configure()
+        if returncode:
+            return
+        returncode = self.build()
+        if returncode:
+            return
+        returncode = self.run(xscope=xscope)
+        if returncode:
+            return
+        time.sleep(5)
+        print("Done!\r")

lib_audio_dsp/python/audio_dsp/design/composite_stage.py

@@ -0,0 +1,213 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""Contains the higher order stage class CompositeStage."""
+
+from typing import Iterable, Type, TypeVar
+
+from .stage import StageOutput, Stage, StageOutputList
+from .graph import Graph
+
+from ._draw import new_record_digraph
+from IPython import display
+from uuid import uuid4
+
+import itertools
+
+_StageOrComposite = TypeVar("_StageOrComposite", bound="Stage | CompositeStage")
+
+
+class CompositeStage:
+    """
+    This is a higher order stage.
+
+    Contains stages as well as other composite
+    stages. A thread will be a composite stage. Composite stages allow:
+
+    - drawing the detail with graphviz
+    - process
+    - frequency response
+
+    Parameters
+    ----------
+    graph : audio_dsp.graph.Graph
+        instance of graph that all stages in this composite will be added to.
+    name : str
+        Name of this instance to use when drawing the pipeline, defaults to class name.
+    """
+
+    # TODO:
+    # - Process method on the composite stage will need to know its inputs and
+    # the order of the inputs (which input index corresponds to each input edge).
+    # However a CompositeStage doesn't know all of its inputs when it is created.
+
+    def __init__(self, graph: Graph, name: str = ""):
+        self._graph = graph
+        self._stages = []
+        self._composite_stages = []
+        self._name = name
+        if not self._name:
+            self._name = type(self).__name__
+
+    def composite_stage(self, name: str = "") -> "CompositeStage":
+        """
+        Create a new composite stage that will be a
+        included in the current composite. The
+        new stage can have stages added to it
+        dynamically.
+        """
+        new = CompositeStage(graph=self._graph, name=name)
+        self._composite_stages.append(new)
+        return new
+
+    @property
+    def o(self) -> StageOutputList:
+        """
+        Outputs of this composite.
+
+        Dynamically computed by searching the graph for edges which
+        originate in this composite and whose destination is outside this composite. Order
+        not currently specified.
+        """
+        all_stages = self.get_all_stages()
+        all_edges = list(itertools.chain.from_iterable([stage.o.edges for stage in all_stages]))
+        return StageOutputList(
+            [edge for edge in all_edges if edge is not None and edge.dest not in all_stages]
+        )
+
+    def stage(
+        self,
+        stage_type: Type[_StageOrComposite],
+        inputs: StageOutputList,
+        label: str | None = None,
+        **kwargs,
+    ) -> _StageOrComposite:
+        """
+        Create a new stage or composite stage and
+        register it with this composite stage.
+
+        Parameters
+        ----------
+        stage_type
+            Must be a subclass of Stage or CompositeStage
+        inputs
+            Edges of the pipeline that will be connected to the newly created stage.
+        kwargs : dict
+            Additional args are forwarded to the stages constructors (__init__)
+
+        Returns
+        -------
+        stage_type
+            Newly created stage or composite stage.
+        """
+        if issubclass(stage_type, CompositeStage):
+            # Subclasses of CompositeStage must have extra __init__
+            # parameters that pyright cant know.
+            stage = stage_type(inputs=inputs, graph=self._graph, **kwargs)  # type: ignore
+            self._composite_stages.append(stage)
+        elif issubclass(stage_type, Stage):
+            stage = stage_type(inputs=inputs, label=label, **kwargs)
+            self._graph.add_node(stage)
+            for edge in stage.o.edges:
+                self._graph.add_edge(edge)
+            self._stages.append(stage)
+        else:
+            raise ValueError(f"{stage_type} is not a Stage or CompositeStage")
+        return stage
+
+    def stages(
+        self, stage_types: list[Type[_StageOrComposite]], inputs: StageOutputList
+    ) -> list[_StageOrComposite]:
+        """
+        Iterate through the provided stages and connect them linearly.
+
+        Returns a list of the created instances.
+        """
+        ret = []
+        for stage_type in stage_types:
+            s = self.stage(stage_type, inputs)
+            ret.append(s)
+            inputs = s.o
+        return ret
+
+    def contains_stage(self, stage: Stage) -> bool:
+        """
+        Recursively search self for the stage.
+
+        Returns
+        -------
+        bool
+            True if this composite contains the stage else False
+        """
+        return stage in self.get_all_stages()
+
+    def get_all_stages(self) -> list[Stage]:
+        """
+        Get a flat list of all stages contained within this composite stage
+        and the composite stages within.
+
+        Returns
+        -------
+            list of stages.
+        """
+        return sum([c.get_all_stages() for c in self._composite_stages], start=self._stages)
+
+    def process(self, data):
+        """
+        Execute the stages in this composite on the host.
+
+        .. warning::
+            Not implemented.
+        """
+        raise NotImplementedError()
+
+    def _internal_edges(self) -> list[StageOutput]:
+        """Return a list of edges whose source and dest are within this composite."""
+        all_stages = self.get_all_stages()
+        all_edges = list(itertools.chain.from_iterable([stage.o.edges for stage in all_stages]))
+        return [
+            edge
+            for edge in all_edges
+            if edge is not None and edge.dest in all_stages and edge.source in all_stages
+        ]
+
+    def draw(self):
+        """Draws the stages and edges present in this instance of a composite stage."""
+        dot = new_record_digraph()
+        self.add_to_dot(dot)
+        output_edges = self.o
+        internal_edges = self._internal_edges()
+        for e in internal_edges:
+            source = f"{e.source.id.hex}:o{e.source_index}:s"
+            dest = f"{e.dest.id.hex}:i{e.dest_index}:n"
+            dot.edge(source, dest)
+
+        end_label = (
+            f"{{ {{ {'|'.join(f'<i{i}> {i}' for i in range(len(output_edges)))} }} | end }}"
+        )
+        dot.node("end", label=end_label)
+        for i, e in enumerate(output_edges):
+            source = f"{e.source.id.hex}:o{e.source_index}:s"
+            dest = f"end:i{i}:n"
+            dot.edge(source, dest)
+        display.display_svg(dot)
+
+    def add_to_dot(self, dot):
+        """
+        Recursively adds composite stages to a dot diagram which is being
+        constructed.
+        Does not add the edges.
+
+        Parameters
+        ----------
+        dot : graphviz.Diagraph
+            dot instance to add edges to.
+        """
+        with dot.subgraph(name=f"cluster_{uuid4().hex}") as subg:
+            subg.attr(color="grey")
+            subg.attr(fontcolor="grey")
+            if self._name:
+                subg.attr(label=self._name)
+            for n in self._stages:
+                n.add_to_dot(subg)
+            for composite_stage in self._composite_stages:
+                composite_stage.add_to_dot(subg)

lib_audio_dsp/python/audio_dsp/design/graph.py

@@ -0,0 +1,160 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+
+"""Basic data structures for managing the pipeline graph."""
+
+from uuid import uuid4
+import graphlib
+from typing import Generic, TypeVar
+
+
+class Node:
+    """
+    Graph node.
+
+    Attributes
+    ----------
+    id : uuid.UUID4
+        A unique identifier for this node.
+    index : None | int
+        node index in the graph. This is set by Graph
+        when it is added to the graph.
+    """
+
+    def __init__(self):
+        self.id = uuid4()
+        self.index: int | None = None
+
+    def __hash__(self) -> int:
+        """Support for using as dictionary/set keys."""
+        return self.id.int
+
+
+class Edge:
+    """
+    Graph node.
+
+    Attributes
+    ----------
+    id : uuid.UUID4
+        A unique identifier for this node.
+    source : Node | None
+    dest : Node | None
+        source and dest are the graph nodes that this edge connects between.
+    """
+
+    def __init__(self):
+        self.id = uuid4()
+        self.source: None | Node = None
+        self.dest: None | Node = None
+
+    def __hash__(self) -> int:
+        """Support for using as dictionary/set keys."""
+        return self.id.int
+
+    def set_source(self, node: Node):
+        """
+        Set the source node of this edge.
+
+        Parameters
+        ----------
+        node
+            The instance to set as the source.
+        """
+        self.source = node
+
+    def set_dest(self, node: Node):
+        """
+        Set the dest node of this edge.
+
+        Parameters
+        ----------
+        node
+            The instance to set as the dest.
+        """
+        self.dest = node
+
+
+NodeSubClass = TypeVar("NodeSubClass", bound=Node)
+
+
+class Graph(Generic[NodeSubClass]):
+    """
+    A container of nodes and edges.
+
+    Attributes
+    ----------
+    nodes
+        A list of the nodes in this graph.
+    edges
+        A list of the edges in this graph.
+    """
+
+    def __init__(self):
+        self.nodes: list[NodeSubClass] = []
+        self.edges: list[Edge] = []
+        self._locked = False
+
+    def add_node(self, node: NodeSubClass) -> None:
+        """
+        Append a node to this graph.
+
+        The node's index attribute is set here and therefore the node may
+        not coexist in multiple graphs.
+        """
+        assert isinstance(node, Node)
+        if self._locked:
+            raise RuntimeError("Cannot add nodes to a locked graph")
+        node.index = len(self.nodes)
+        self.nodes.append(node)
+
+    def add_edge(self, edge) -> None:
+        """Append an edge to this graph."""
+        assert isinstance(edge, Edge)
+        if self._locked:
+            raise RuntimeError("Cannot add edges to a locked graph")
+        self.edges.append(edge)
+
+    def get_view(self, nodes: list[NodeSubClass]) -> "Graph[NodeSubClass]":
+        """
+        Get a filtered view of the graph, including only the provided nodes and the
+        edges which connect to them.
+        """
+        ret = Graph()
+        ret.nodes = nodes
+        ret.edges = [e for e in self.edges if e.source in nodes or e.dest in nodes]
+        return ret
+
+    def get_dependency_dict(self) -> dict[NodeSubClass, set[NodeSubClass]]:
+        """
+        Return a mapping of nodes to their dependencies ready for use with the graphlib
+        utilities.
+        """
+        graph = {}
+        for node in self.nodes:
+            graph[node] = set()
+
+        for edge in self.edges:
+            if edge.dest in graph and edge.source is not None:
+                graph[edge.dest].add(edge.source)
+        return graph
+
+    def sort(self) -> tuple[NodeSubClass, ...]:
+        """
+        Sort the nodes in the graph based on the order they should be executed.
+        This is determined by looking at the edges in the graph and resolving the
+        order.
+
+        Returns
+        -------
+        tuple[Node]
+            Ordered list of nodes
+        """
+        return tuple(graphlib.TopologicalSorter(self.get_dependency_dict()).static_order())
+
+    def lock(self):
+        """
+        Lock the graph. Adding nodes or edges to a locked graph will cause a runtime exception.
+        The graph is locked once the pipeline checksum is computed.
+        """
+        self._locked = True

lib_audio_dsp/python/audio_dsp/design/parse_config.py

@@ -0,0 +1,110 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""
+Script for use at build time to generate header files.
+
+Use as::
+
+    python -m audio_dsp.design.parse_config -c CONFIG_DIR -o OUTPUT_DIR
+"""
+
+import yaml
+import glob
+from mako.template import Template
+import argparse
+import os
+from pathlib import Path
+
+
+def main(args):
+    """Use the mako templates to build the autogenerated files."""
+    pkg_dir = Path(__file__).parent
+    templates_dir = f"{pkg_dir}/templates"
+
+    os.makedirs(args.out_dir, exist_ok=True)
+    os.makedirs(f"{args.out_dir}/common", exist_ok=True)
+    os.makedirs(f"{args.out_dir}/device", exist_ok=True)
+    os.makedirs(f"{args.out_dir}/host", exist_ok=True)
+    os.makedirs(f"{args.out_dir}/generator", exist_ok=True)
+
+    struct_def_template = Template(filename=f"{templates_dir}/struct_def_h.mako")
+    struct_offset_template = Template(filename=f"{templates_dir}/gen_command_map_offsets_c.mako")
+    cmd_map_template = Template(filename=f"{templates_dir}/host_cmd_map.mako")
+    cmd_ids_template = Template(filename=f"{templates_dir}/cmds_h.mako")
+    module_config_offsets_template = Template(filename=f"{templates_dir}/cmd_offsets_h.mako")
+
+    cmd_map = {}
+
+    files = glob.glob(f"{args.config_dir}/*.yaml")
+    for fl in files:
+        with open(fl, "r") as fd:
+            data = yaml.safe_load(fd)
+
+            struct_name = list(data["module"].keys())[0]
+            includes = data["includes"] if "includes" in data else []
+            defines = data["defines"] if "defines" in data else dict()
+
+            for param in list(data["module"][struct_name]):
+                if "help" not in data["module"][struct_name][param]:
+                    raise ValueError(
+                        f"{param} in {struct_name} in {fl} does not contain a help parameter"
+                    )
+                elif "*" in data["module"][struct_name][param]["help"]:
+                    raise ValueError(
+                        f"{param} in {struct_name} in {fl} contains a forbidden character '*'"
+                    )
+
+            with open(f"{args.out_dir}/common/{struct_name}_config.h", "w", newline="") as f_op:
+                f_op.write(
+                    struct_def_template.render(
+                        name=struct_name,
+                        data=data["module"][struct_name],
+                        includes=includes,
+                        defines=defines,
+                    )
+                )
+
+            cmd_map[struct_name] = data["module"][struct_name]
+
+    cmd_map = dict(sorted(cmd_map.items()))
+    with open(f"{args.out_dir}/generator/gen_cmd_map_offset.c", "w", newline="") as f_op:
+        f_op.write(struct_offset_template.render(cmd_map=cmd_map))
+    # Generate cmd_map used by the host
+    with open(f"{args.out_dir}/host/host_cmd_map.h", "w", newline="") as f_op:
+        f_op.write(cmd_map_template.render(cmd_map=cmd_map))
+
+    # Generate #defines present in the cmd map used by the host
+    with open(f"{args.out_dir}/common/cmds.h", "w", newline="") as f_op:
+        f_op.write(cmd_ids_template.render(cmd_map=cmd_map))
+
+    # generate the config offsets for every module
+    with open(f"{args.out_dir}/device/cmd_offsets.h", "w") as f_op:
+        f_op.write(module_config_offsets_template.render(cmd_map=cmd_map))
+
+
+if __name__ == "__main__":
+
+    def parse_arguments():
+        """Parse command line arguments."""
+        parser = argparse.ArgumentParser(description="Generate config struct files")
+        parser.add_argument(
+            "--config-dir",
+            "-c",
+            required=True,
+            type=str,
+            default=None,
+            help="Path to the config yaml files",
+        )
+        parser.add_argument(
+            "--out-dir",
+            "-o",
+            type=str,
+            default=f"{str(Path.cwd())}/output_files",
+            help="output directory for host and device files",
+        )
+        args = parser.parse_args()
+        return args
+
+    ## Parse command line arguments
+    args = parse_arguments()
+    main(args)

lib_audio_dsp/python/audio_dsp/design/parse_json.py

@@ -0,0 +1,372 @@
+# Copyright 2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""Functions to convert JSON files to Python DSP pipelines."""
+
+from __future__ import annotations
+from collections import defaultdict
+from pathlib import Path
+from pprint import pprint
+from typing import Annotated, Any, Optional, Union, TypeVar, TypeAlias
+
+from pydantic import BaseModel, Field
+from pydantic.json_schema import SkipJsonSchema
+
+import audio_dsp.stages as Stages
+from audio_dsp.design.pipeline import Pipeline, generate_dsp_main
+from audio_dsp.design.stage import StageOutputList, all_stages
+from audio_dsp.models.signal_chain import Fork
+from audio_dsp.models.stage import all_models, StageModel
+import json
+import re
+import warnings
+
+BAD_NAMES = []
+
+# Define the union type alias for all stage models
+_stage_Models = Annotated[
+    Union[
+        tuple((SkipJsonSchema[i] if i.__name__ in BAD_NAMES else i) for i in all_models().values())
+    ],
+    Field(discriminator="op_type"),
+]
+
+
+class Input(BaseModel, extra="ignore"):
+    """Pydantic model of the inputs to a DSP graph."""
+
+    name: str = Field(..., description="Name of the input")
+    channels: int
+
+
+class Output(BaseModel, extra="ignore"):
+    """Pydantic model of the outputs of a DSP graph."""
+
+    name: str = Field(..., description="Name of the output")
+    input: list[tuple[str, int]] = Field(
+        ...,
+        description="List of input edges as (node_name, index) tuples",
+    )
+
+
+StageModelType = TypeVar("StageModelType", bound=StageModel)
+
+stage_models_list: TypeAlias = list[_stage_Models]  # pyright: ignore
+
+
+#
+class Graph(BaseModel):
+    """Graph object to hold the pipeline information."""
+
+    name: str = Field(..., description="Name of the graph")
+    fs: int = Field(..., description="Sampling frequency for the graph")
+    nodes: stage_models_list  # pyright: ignore
+    inputs: list[Input]
+    outputs: list[Output]
+
+
+def path_encoder(obj):
+    """Encode Path objects as strings for JSON serialization."""
+    if isinstance(obj, Path):
+        return str(obj)
+    raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+
+class DspJson(BaseModel):
+    """Pydantic model of the JSON file describing a DSP graph."""
+
+    ir_version: int
+    producer_name: str
+    producer_version: str
+    graph: Graph
+    # checksum: List
+
+    def model_dump_xdsp(self, indent=2):
+        """Dump model in xdsp format with specified indentation."""
+        d = self.model_dump()
+        # Move 'op_type' to the front
+        for i, node in enumerate(d["graph"]["nodes"]):
+            if "op_type" in node:
+                items = [("op_type", node.pop("op_type"))] + list(node.items())
+                d["graph"]["nodes"][i] = dict(items)
+                # return json.dumps(dict(items))
+        dump = json.dumps(d, indent=2, default=path_encoder)
+
+        def compact_array_newlines(s):
+            # Finds square brackets with their content (non-greedy, so not nested)
+            pattern = re.compile(r"\[(.*?)\]", re.DOTALL)
+
+            def repl(match):
+                inner = match.group(1)
+                if "[" in inner:
+                    # catch the first node
+                    return f"[{compact_array_newlines(inner + ']')}"
+                elif "{" in inner or "}" in inner:
+                    # catch other lists
+                    return match.group(0)
+                # Remove leading/trailing whitespace just inside the brackets
+                stripped_inner = inner.strip()
+                # Replace inner newlines+whitespace with a space
+                compact_inner = re.sub(r"\s*\n\s*", " ", stripped_inner)
+                return f"[{compact_inner}]"
+
+            return pattern.sub(repl, s)
+
+        return compact_array_newlines(dump)
+
+
+def insert_forks(graph: Graph) -> Graph:
+    """Automatically insert forks in the graph where edges have been used
+    multiple times.
+    """
+    # Create a deep copy of the graph to avoid mutating the original
+    new_graph = graph.model_copy(deep=True)
+
+    # Map from edge name to a list of consumers (node or graph output, index, position)
+    consumer_map: dict[tuple[str, int], list[tuple[str, int, int]]] = defaultdict(list)
+
+    # Build the consumer map for all node inputs
+    for node_index, node in enumerate(new_graph.nodes):
+        for input_idx, edge_tuple in enumerate(node.placement.input):
+            consumer_map[edge_tuple].append(("node", node_index, input_idx))
+
+    # Add graph outputs as consumers
+    for out_idx, out in enumerate(new_graph.outputs):
+        for input_idx, edge_tuple in enumerate(out.input):
+            consumer_map[edge_tuple].append(("graph_output", out_idx, input_idx))
+
+    node_dict = {node.placement.name: i for i, node in enumerate(graph.nodes)}
+
+    autofork_idx = 0
+    for key in consumer_map:
+        # If there are multiple consumers for the same edge, we need to insert a Fork node
+        if len(consumer_map[key]) > 1:
+            # calculate the fork properties
+            num_consumers = len(consumer_map[key])
+            first_node_idx = consumer_map[key][0][1] + autofork_idx
+            if key[0] == "inputs":
+                # If the edge is an input, we can use the first consumer's thread
+                thread = new_graph.nodes[first_node_idx].placement.thread
+            else:
+                # Otherwise, put the fork on the producer thread
+                key_idx = node_dict[key[0]]
+                thread = new_graph.nodes[key_idx].placement.thread
+
+            # Create the Fork node
+            fork_node_data = {
+                "op_type": "Fork",
+                "config": {"count": num_consumers},
+                "placement": {
+                    "input": [key],
+                    "name": f"AutoFork_{autofork_idx}",
+                    "thread": thread,
+                },
+            }
+
+            # add the Fork node to the new graph where the first node was
+            new_graph.nodes.insert(first_node_idx, Fork(**fork_node_data))
+            autofork_idx += 1
+
+            # update the consumers to use the new fork node
+            for i, cons in enumerate(consumer_map[key]):
+                cons_type, consumer_idx, pos = cons
+                new_edge_name = (f"AutoFork_{autofork_idx - 1}", i)
+                # Update the consumer map to point to the new edge name,
+                if cons_type == "node":
+                    # compmensating for the inserted autofork index
+                    new_graph.nodes[consumer_idx + autofork_idx].placement.input[pos] = (
+                        new_edge_name
+                    )
+                elif cons_type == "graph_output":
+                    new_graph.outputs[consumer_idx].input[pos] = new_edge_name
+
+        else:
+            # If there's only one consumer, we can just keep the original edge
+            continue
+
+    return new_graph
+
+
+def stage_handle(model):
+    """Get the function handle of a DSP Stage from its pydantic model."""
+    return getattr(Stages, model.op_type)
+
+
+def make_pipeline(json_obj: DspJson) -> Pipeline:
+    """Create a Python DSP pipeline from a Pydantic model of the JSON file
+    describing a DSP graph.
+    """
+    # Insert Fork nodes where needed to handle shared edges
+    graph = insert_forks(json_obj.graph)
+    edge_map = {}  # edge_name (tuple) -> pipeline object
+
+    # Collect all input edge names and total channels
+    flat_input_edges = []
+    total_channels = 0
+    for inp in graph.inputs:
+        flat_input_edges.extend([(inp.name, i) for i in range(inp.channels)])
+        total_channels += inp.channels
+
+    p, in_edges = Pipeline.begin(total_channels, fs=graph.fs, identifier=graph.name)
+    thread_max = max(node.placement.thread for node in graph.nodes) if graph.nodes else 0
+    for _ in range(thread_max):
+        p._add_thread()
+
+    # Assign input edges
+    for i, edge_name in enumerate(flat_input_edges):
+        edge_map[edge_name] = in_edges[i]
+
+    waiting_nodes = list(range(len(graph.nodes)))
+    waiting_count = len(waiting_nodes)
+    waiting_timer = 0
+    while waiting_nodes:
+        node_idx = waiting_nodes[0]
+        node = graph.nodes[node_idx]
+        # Gather StageOutput objects for this node's inputs
+        stage_inputs = []
+        for edge_name in node.placement.input:
+            stage_inputs.append(edge_map.get(tuple(edge_name)))
+        if None in stage_inputs:
+            # use a timer to avoid getting stuck
+            if len(waiting_nodes) < waiting_count:
+                waiting_count = len(waiting_nodes)
+                waiting_timer = 0
+            else:
+                waiting_timer += 1
+                if waiting_timer > waiting_count:
+                    raise ValueError(
+                        f"Node {node_idx} ({node.placement.name}) has inputs that are not ready: {stage_inputs}"
+                    )
+            waiting_nodes.pop(0)
+            waiting_nodes.append(node_idx)
+            continue
+        # Combine all inputs into a StageOutputList
+        stage_inputs = sum(stage_inputs, start=StageOutputList())
+        # Get config dict for this node
+        config = node.config if hasattr(node, "config") else {}
+        if isinstance(config, BaseModel):
+            config = config.model_dump()
+        # Instantiate the stage and get its outputs
+        node_output = p.stage(
+            stage_handle(node),
+            stage_inputs,
+            node.placement.name,
+            thread=node.placement.thread,
+            **config,
+        )
+        # Set parameters if present
+        if hasattr(node, "parameters"):
+            p.stages[-1].set_parameters(node.parameters)
+        # Map the outputs to the correct edge indices
+        if len(node_output) != 0:
+            for i in range(len(node_output)):
+                edge_name = (node.placement.name, i)
+                if edge_map.get(edge_name) is not None:
+                    raise ValueError("Output already exists")
+                edge_map[edge_name] = node_output[i]
+        # Remove this node from waiting list
+        waiting_nodes.pop(0)
+    # Gather all output nodes for the pipeline
+    output_nodes = []
+    for edge_name in [tuple(e) for out in graph.outputs for e in out.input]:
+        output_nodes.append(edge_map[edge_name])
+    output_nodes = sum(output_nodes, start=StageOutputList())
+    # Set the pipeline outputs
+    p.set_outputs(output_nodes)
+    return p
+
+
+def update_pipeline(p: Pipeline, params: DspJson):
+    """Update the pipeline with new DSP JSON parameters."""
+    for stage in p.stages[1:]:
+        if stage.name in ["pipeline", "dsp_thread"]:
+            continue
+        updated = False
+        for node in params.graph.nodes:
+            if node.placement.name == stage.label:
+                updated = True
+                if hasattr(node, "parameters"):
+                    stage.set_parameters(node.parameters)
+                break
+
+        if not updated and "AutoFork" not in stage.label:
+            warnings.warn(f"Stage {stage.label} could not be found in the JSON file")
+
+
+def pipeline_to_dspjson(pipeline) -> DspJson:
+    """Convert a Pipeline object to a DspJson object."""
+    # Example: Extract graph-level info
+    graph_name = getattr(pipeline, "_id")
+    fs = getattr(pipeline, "fs")
+
+    # Extract inputs and outputs
+    inputs = [
+        Input(
+            name="inputs",
+            channels=len(pipeline.i.edges),
+        )
+    ]
+
+    output_in = []
+    for x in pipeline.o.edges:
+        if x.source is not None:
+            output_in.append([f"{x.source.label}", x.source_index])
+        else:
+            # if the source is None, it means it's a direct input to the pipeline
+            output_in.append([f"inputs", x.source_index])
+
+    outputs = [Output(name="outputs", input=output_in)]
+
+    # Extract nodes
+    nodes = []
+    for thread in pipeline.threads:
+        for stage in thread._stages:
+            op_type = type(stage).__name__
+            if op_type in ["PipelineStage", "DSPThreadStage"]:
+                continue
+
+            stage_in = []
+            for x in stage.i.edges:
+                if x.source is not None:
+                    stage_in.append([f"{x.source.label}", x.source_index])
+                else:
+                    # if the source is None, it means it's a direct input to the pipeline
+                    stage_in.append([f"inputs", x.source_index])
+
+            placement = {
+                "name": stage.label or stage.name,
+                "input": stage_in,
+                "thread": thread.id,
+            }
+
+            node_dict = {
+                "op_type": op_type,
+                "placement": placement,  # Should be a dict or Pydantic model
+            }
+            if hasattr(stage, "config"):
+                node_dict["config"] = stage.config
+            if hasattr(stage, "parameters"):
+                node_dict["parameters"] = stage.parameters
+            # Convert to the correct Pydantic model for the node
+            node_model_cls = type(stage.model) if hasattr(stage, "model") else None
+            if node_model_cls:
+                node = node_model_cls(**node_dict)
+            else:
+                node = node_dict  # fallback, but ideally use the model
+            nodes.append(node)
+
+    graph = Graph(
+        name=graph_name,
+        fs=fs,
+        nodes=nodes,
+        inputs=inputs,
+        outputs=outputs,
+    )
+
+    # Fill in DspJson fields
+    dsp_json = DspJson(
+        ir_version=1,
+        producer_name="pipeline_to_dspjson",
+        producer_version="1.0",
+        graph=graph,
+    )
+    return dsp_json

lib_audio_dsp/python/audio_dsp/design/pipeline_executor.py

@@ -0,0 +1,271 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""Utilities for processing the pipeline on the host machine."""
+
+from pathlib import Path
+from typing import Optional, NamedTuple
+from collections.abc import Callable
+from scipy.io import wavfile
+from matplotlib import pyplot as plt
+
+import numpy
+from IPython import display
+
+
+from .stage import Stage, StageOutput
+from .graph import Graph
+from ..dsp import signal_gen
+
+
+class PipelineView(NamedTuple):
+    """A view of the DSP pipeline that is used by PipelineExecutor."""
+
+    stages: Optional[list[Stage]]
+    inputs: list[StageOutput]
+    outputs: list[StageOutput]
+
+
+class ExecutionResult:
+    """
+    The result of processing samples through the pipeline.
+
+    Parameters
+    ----------
+    result
+        The data produced by the pipeline.
+    fs
+        sample rate
+
+    Attributes
+    ----------
+    data
+        ndarray containing the results of the pipeline.
+    fs
+        Sample rate.
+    """
+
+    def __init__(self, result: numpy.ndarray, fs: float):
+        self.data = result
+        self.fs = fs
+
+    def to_wav(self, path: str | Path):
+        """Save output to a wav file."""
+        wavfile.write(path, self.fs, self.data)
+
+    def plot(self, path: Optional[str | Path] = None):
+        """
+        Display a time domain plot of the result. Save to file
+        if path is not None.
+
+        Parameters
+        ----------
+        path
+            If path is not none then the plot will be saved to a file
+            and not shown.
+        """
+        chans = self.data.shape[1]
+        period = 1 / self.fs
+        t = numpy.arange(0, self.data.shape[0] * period, period)
+        fig, axs = plt.subplots(chans, sharex=True, squeeze=False)
+        for i in range(chans):
+            axs[i][0].plot(t, self.data[:, i])
+        if path:
+            plt.savefig(path)
+        else:
+            plt.show()
+
+    def plot_magnitude_spectrum(self, path: Optional[str | Path] = None):
+        """
+        Display a spectrum plot of the result. Save to file
+        if path is not None.
+
+        Parameters
+        ----------
+        path
+            If path is not none then the plot will be saved to a file
+            and not shown.
+        """
+        chans = self.data.shape[1]
+        fig, axs = plt.subplots(chans, sharex=True, squeeze=False)
+        for i in range(chans):
+            axs[i][0].magnitude_spectrum(self.data[:, i], Fs=self.fs, scale="dB")
+        if path:
+            plt.savefig(path)
+        else:
+            plt.show()
+
+    def plot_spectrogram(self, path: Optional[str | Path] = None):
+        """
+        Display a spectrogram plot of the result. Save to file
+        if path is not None.
+
+        Parameters
+        ----------
+        path
+            If path is not none then the plot will be saved to a file
+            and not shown.
+        """
+        chans = self.data.shape[1]
+        fig, axs = plt.subplots(chans, sharex=True, squeeze=False)
+        for i in range(chans):
+            Pxx, _, _, _ = axs[i][0].specgram(self.data[:, i], NFFT=1024, Fs=self.fs, noverlap=900)
+            Pxx_max_dB = 10 * numpy.log10(Pxx.max()) - 5
+            axs[i][0].specgram(
+                self.data[:, i],
+                NFFT=1024,
+                Fs=self.fs,
+                noverlap=900,
+                cmap=plt.get_cmap("cividis"),
+                vmin=Pxx_max_dB - 50,
+                vmax=Pxx_max_dB,
+            )
+        fig.supylabel("Frequency [Hz]")
+        fig.supxlabel("Time [sec]")
+        if path:
+            plt.savefig(path)
+        else:
+            plt.show()
+
+    def play(self, channel: int):
+        """
+        Create a widget in the jupyter notebook to listen to the audio.
+
+        .. warning::
+            This will not work outside of a jupyter notebook.
+
+        Parameters
+        ----------
+        channel
+            The channel to listen to.
+        """
+        display.display(display.Audio(self.data[:, channel], rate=int(self.fs)))
+
+
+class PipelineExecutor:
+    """
+    Utility for simulating the pipeline.
+
+    Parameters
+    ----------
+    graph
+        The pipeline graph to simulate
+    """
+
+    def __init__(self, graph: Graph[Stage], view_getter: Callable[[], PipelineView]):
+        self._graph = graph
+        self._view_getter = view_getter
+
+    def _get_view(self) -> tuple[Graph[Stage], list[StageOutput], list[StageOutput]]:
+        view = self._view_getter()
+        graph = self._graph.get_view(view.stages) if view.stages else self._graph
+        return graph, view.inputs, view.outputs
+
+    def _process_frame(
+        self,
+        frame: list[numpy.ndarray],
+        graph: Graph[Stage],
+        i_edges: list[StageOutput],
+        o_edges: list[StageOutput],
+    ) -> dict[int, numpy.ndarray]:
+        """Process channels through the pipeline and return the result."""
+        edges = {}
+        for edge, data in zip(i_edges, frame):
+            edges[edge] = data
+
+        stages = graph.sort()
+
+        for stage in stages:
+            inputs = [edges[e] for e in stage.i.edges]
+            if stage.o or stage.i:
+                outputs = stage.process(inputs)
+                edges.update({e: o for e, o in zip(stage.o.edges, outputs)})
+
+        return {i: edges[e] for i, e in enumerate(o_edges) if e is not None}
+
+    def process(self, data: numpy.ndarray) -> ExecutionResult:
+        """
+        Process the DSP pipeline on the host.
+
+        Parameters
+        ----------
+        data
+            Pipeline input to process through the pipeline. The shape must match the number of channels
+            that the pipeline expects as an input; if this is 1 then it may be a 1 dimensional array. Otherwise,
+            it must have shape (num_samples, num_channels).
+
+        Returns
+        -------
+        ExecutionResult
+            A result object that can be used to visualise or save the output.
+        """
+        graph, i_edges, o_edges = self._get_view()
+        n_i_chans = len(i_edges)
+        n_o_chans = len(o_edges)
+
+        if len(data.shape) == 1:
+            data = data.reshape((data.shape[0], 1))
+        if len(data.shape) != 2:
+            raise ValueError("Can only process 2D or 1D arrays of inputs")
+        if data.shape[1] != n_i_chans:
+            raise ValueError(f"Received {data.shape[1]} channels, expected {n_i_chans}")
+
+        # Assume all edges have the same frame size
+        frame_size = i_edges[0].frame_size
+        fs = i_edges[0].fs
+
+        if data.shape[0] % frame_size:
+            # truncate data to a multiple of frame size
+            datalen = (data.shape[0] // frame_size) * frame_size
+            data = data[:datalen, :]
+
+        ret = numpy.zeros((data.shape[0], n_o_chans))
+
+        for index in range(0, data.shape[0], frame_size):
+            inputs = [data[index : (index + frame_size), i] for i in range(n_i_chans)]
+            outputs = self._process_frame(inputs, graph, i_edges, o_edges)
+            for i, val in outputs.items():
+                ret[index : index + frame_size, i] = val
+        return ExecutionResult(ret, fs)
+
+    def log_chirp(
+        self,
+        length_s: float = 0.5,
+        amplitude: float = 1,
+        start: float = 20,
+        stop: Optional[float] = None,
+    ) -> ExecutionResult:
+        """
+        Generate a logarithmic chirp of constant amplitude and play through
+        the simulated pipeline.
+
+        Parameters
+        ----------
+        length_s
+            Length of generated chirp in seconds.
+        amplitude
+            Amplitude of the generated chirp, between 0 and 1.
+        start
+            Start frequency.
+        stop
+            Stop frequency. Nyquist if not set
+
+        Returns
+        -------
+        ExecutionResult
+            The output wrapped in a helpful container for viewing, saving, processing, etc.
+        """
+        _, i_edges, _ = self._get_view()
+        fs = i_edges[0].fs
+        if fs is None:
+            raise RuntimeError(
+                "Executor makes the assumption that the pipeline edges all have fs set"
+            )
+        frame_size = i_edges[0].frame_size
+
+        chirp = signal_gen.log_chirp(fs, length_s, amplitude, start=start, stop=stop or fs / 2)
+        chirp_len = len(chirp)
+        desired_chirp_len = chirp_len - (chirp_len % frame_size)
+        chirp = chirp[:desired_chirp_len]
+        chirp = numpy.stack([chirp] * len(i_edges), axis=1)
+
+        return self.process(chirp)

lib_audio_dsp/python/audio_dsp/design/plot.py

@@ -0,0 +1,51 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""
+Helper functions for displaying plots in the jupyter notebook pipeline
+design.
+"""
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+import audio_dsp.dsp.utils as utils
+
+
+def plot_frequency_response(f, h, name="", range=50):
+    """
+    Plot the frequency response.
+
+    Parameters
+    ----------
+    f : numpy.ndarray
+        Frequencies (The X axis)
+    h : numpy.ndarray
+        Frequency response at the corresponding frequencies in ``f``
+    name : str
+        String to include in the plot title, if not set there will be no title.
+    range : int | float
+        Set the Y axis lower limit in dB, upper limit will be the maximum
+        magnitude.
+    """
+    h_db = utils.db(h)
+
+    y_max = np.max(h_db) + 1
+    y_min = y_max - range
+
+    fig, axs = plt.subplots(2, 1, sharex=True)
+    if name:
+        fig.suptitle(f"{name} frequency response".title())
+    axs[0].semilogx(f, h_db)
+    axs[0].set_ylim([y_min, y_max])
+    axs[0].set_xlim([20, 20000])
+    axs[0].set_ylabel("Magnitude (dB)")
+    axs[0].grid()
+
+    axs[1].semilogx(f, np.angle(h))
+    axs[1].set_ylabel("Phase (rad)")
+    axs[1].grid()
+    axs[1].set_xlabel("Frequency (Hz)")
+
+    plt.show()
+
+    return fig

lib_audio_dsp/python/audio_dsp/design/stage.py

@@ -0,0 +1,620 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""The edges and nodes for a DSP pipeline."""
+
+from pathlib import Path
+from types import NotImplementedType
+from typing import Optional, Type, TypeVar
+
+import numpy
+import yaml
+import re
+
+from audio_dsp.design import plot
+from audio_dsp.dsp.generic import dsp_block
+from typing import Optional
+from types import NotImplementedType
+from copy import deepcopy
+
+from audio_dsp.models.stage import StageParameters
+from audio_dsp.design.graph import Edge, Node
+
+
+def find_config(name):
+    """
+    Find the config yaml file for a stage by looking for it
+    in the default directory for built in stages.
+
+    Parameters
+    ----------
+    name : str
+        Name of stage, e.g. a stage whose config is saved in "biquad.yaml"
+        should pass in "biquad".
+
+    Returns
+    -------
+    Path
+        Path to the config file.
+    """
+    ret = Path(__file__).parents[3] / "stage_config" / f"{name}.yaml"
+    if not ret.exists():
+        raise ValueError(f"{ret} does not exist")
+    return ret
+
+
+class StageOutput(Edge):
+    """
+    The Edge of a dsp pipeline.
+
+    Parameters
+    ----------
+    fs : int
+        Edge sample rate Hz
+    frame_size : int
+        Number of samples per frame
+
+    Attributes
+    ----------
+    source : audio_dsp.design.graph.Node
+        Inherited from Edge
+    dest : audio_dsp.design.graph.Node
+        Inherited from Edge
+    source_index : int | None
+        The index of the edge connection to source.
+    fs : int
+        see fs parameter
+    frame_size : int
+        see frame_size parameter
+    """
+
+    def __init__(self, fs=48000, frame_size=1, crossings=None):
+        super().__init__()
+        # index of the multiple outputs that the source node has
+        self.source_index = None
+        # which input index is this
+        self._dest_index = None
+        self.fs = fs
+        self.frame_size = frame_size
+        # edges will probably need an associated type audio vs. data etc.
+        # self.type = q23
+        if crossings:
+            self.crossings = deepcopy(crossings)
+        else:
+            self.crossings = []
+
+    @property
+    def dest_index(self) -> int | None:
+        """The index of the edge connection to the dest."""
+        return self._dest_index
+
+    @dest_index.setter
+    def dest_index(self, value):
+        if self._dest_index is not None:
+            raise RuntimeError(
+                f"This edge has already been connected, edges cannot have multiple destinations: {value = }."
+            )
+        self._dest_index = value
+
+    def __repr__(self) -> str:
+        """Make print output usable."""
+        dest = "-" if self.dest is None else f"{self.dest.index} {self.dest_index}"
+        source = "-" if self.source is None else f"{self.source.index} {self.source_index}"
+        return f"({source} -> {dest})"
+
+
+class StageOutputList:
+    """
+    A container of StageOutput.
+
+    A stage output list will be created whenever a stage is added to the pipeline.
+    It is unlikely that a StageOutputList will have to be explicitly created during pipeline
+    design. However the indexing and combining methods shown in the example will be used to
+    create new StageOutputList instances.
+
+    Examples
+    --------
+    This example shows how to combine StageOutputList in various ways::
+
+        # a and b are StageOutputList
+        a = some_stage.o
+        b = other_stage.o
+
+        # concatenate them
+        a + b
+
+        # Choose a single channel from 'a'
+        a[0]
+
+        # Choose channels 0 and 3 from 'a'
+        a[0, 3]
+
+        # Choose a slice of channels from 'a', start:stop:step
+        a[0:10:2]
+
+        # Combine channels 0 and 3 from 'a', and 2 from 'b'
+        a[0, 3] + b[2]
+
+        # Join 'a' and 'b', with a placeholder "None" in between
+        a + None + b
+
+    Attributes
+    ----------
+    edges : list[StageOutput]
+        To access the actual edges contained within this list then read from the edges
+        attribute. All methods in this class return new StageOutputList instances (even
+        when the length is 1).
+
+    Parameters
+    ----------
+    edges
+        list of StageOutput to create this list from.
+    """
+
+    def __init__(self, edges: list[StageOutput | None] | None = None):
+        edges = edges or []
+        for edge in edges:
+            if not isinstance(edge, StageOutput) and edge is not None:
+                raise TypeError(
+                    f"Expected iterable of StageOutput or None, however it contained a {type(edge)}"
+                )
+        self.edges = edges
+
+    def __iter__(self):
+        """Iterate through the edges, yielding a new StageOutputList for each edge."""
+        for e in self.edges:
+            yield StageOutputList([e])
+
+    def __len__(self):
+        """Get the number of edges in this list."""
+        return len(self.edges)
+
+    def __radd__(self, other) -> "StageOutputList | NotImplementedType":
+        """Other + self."""
+        if isinstance(other, list):
+            other = StageOutputList(other)
+        if other is None:
+            other = StageOutputList([None])
+        if not isinstance(other, StageOutputList):
+            return NotImplemented
+        return StageOutputList(other.edges + self.edges)
+
+    def __add__(self, other) -> "StageOutputList | NotImplementedType":
+        """Create a new StageOutputList which concatenates the input lists."""
+        if isinstance(other, list):
+            other = StageOutputList(other)
+        if other is None:
+            other = StageOutputList([None])
+        if not isinstance(other, StageOutputList):
+            return NotImplemented
+        return StageOutputList(self.edges + other.edges)
+
+    def __or__(self, other) -> "StageOutputList":
+        """Support for '|', does the same as __add__."""
+        return self + other
+
+    def __ror__(self, other) -> "StageOutputList":
+        """Support for '|', does the same as __radd__."""
+        return other + self
+
+    def __getitem__(self, key) -> "StageOutputList":
+        """Create new StageOutputList containing requested indices from this one."""
+        if isinstance(key, slice):
+            return StageOutputList(self.edges[key])
+        elif isinstance(key, int):
+            return StageOutputList([self.edges[key]])
+        else:
+            return StageOutputList([self.edges[i] for i in key])
+
+    def __eq__(self, other):
+        """Check if this list contains the same edges as another."""
+        if other is None:
+            return False
+        else:
+            return all(a is b for a, b in zip(self.edges, other.edges))
+
+
+class PropertyControlField:
+    """For stages which have internal state they can register callbacks
+    for getting and setting control fields.
+    """
+
+    def __init__(self, get, set=None):
+        self._getter = get
+        self._setter = set
+
+    @property
+    def value(self):
+        """
+        The current value of this control field.
+
+        Determined by executing the getter method.
+        """
+        return self._getter()
+
+    @value.setter
+    def value(self, value):
+        if self._setter is None:
+            raise RuntimeError("This control field can't be set directly")
+        self._setter(value)
+
+
+class ValueControlField:
+    """Simple field which can be updated directly."""
+
+    def __init__(self, value=None):
+        self.value = value
+
+
+class _GlobalStages:
+    """Class to hold some globals."""
+
+    stages = []
+
+
+# This defines the types of instances of the config/parameter classes
+StageParameterType = TypeVar("StageParameterType", bound=StageParameters)
+DspBlockType = TypeVar("DspBlockType", bound=dsp_block)
+
+
+class Stage(Node):
+    """
+    Base class for stages in the DSP pipeline. Each subclass
+    should have a corresponding C implementation. Enables
+    code generation, tuning and simulation of a stage.
+
+    The stages config can be written and read using square brackets as with a
+    dictionary. This is shown in the below example, note that the config field
+    must have been declared in the stages yaml file.
+
+        self["config_field"] = 2
+        assert self["config_field"] == 2
+
+    Parameters
+    ----------
+    config : str | Path
+        Path to yaml file containing the stage definition for this stage. Config
+        parameters are derived from this config file.
+    inputs : Iterable[StageOutput]
+        Pipeline edges to connect to self
+    name : str
+        Name of the stage. Passed instead of config when the stage does not have
+        an associated config yaml file
+    label : str
+        User defined label for the stage. Used for autogenerating a define for accessing the stage's index
+        in the device code
+
+    Attributes
+    ----------
+    i : list[StageOutput]
+        This stages inputs.
+    fs : int | None
+        Sample rate.
+    frame_size : int | None
+        Samples in frame.
+    name : str
+        Stage name determined from config file
+    yaml_dict : dict
+        config parsed from the config file
+    label : str
+        User specified label for the stage
+    n_in : int
+        number of inputs
+    n_out : int
+        number of outputs
+    details : dict
+        Dictionary of descriptive details which can be displayed to describe
+        current tuning of this stage
+    dsp_block : None | audio_dsp.dsp.generic.dsp_block
+        This will point to a dsp block class (e.g. biquad etc), to be set by the child class
+    """
+
+    def __init__(
+        self,
+        inputs: StageOutputList,
+        config: Optional[Path | str] = None,
+        name: Optional[str] = None,
+        label: Optional[str] = None,
+    ):
+        super().__init__()
+        self.i = inputs[:]
+        for i, input in enumerate(self.i.edges):
+            if input is None:
+                raise TypeError("All stage inputs must not be None")
+            input.set_dest(self)
+            input.dest_index = i
+        if self.i:
+            assert self.i.edges[0] is not None, "not possible as checked above"
+            self.fs = self.i.edges[0].fs
+            self.frame_size = self.i.edges[0].frame_size
+        else:
+            self.fs = None
+            self.frame_size = None
+
+        self.n_in = len(self.i)
+        self.n_out = 0
+        self._o = None
+        if (config is None and name is None) or (config is not None and name is not None):
+            raise RuntimeError("Provide either config or name, not both or none.")
+        if config is not None:
+            self.yaml_dict = yaml.load(Path(config).read_text(), Loader=yaml.Loader)
+            # module dict contains 1 entry with the name of the module as its key
+            self.name = next(iter(self.yaml_dict["module"].keys()))
+            self._control_fields = {
+                name: ValueControlField() for name in self.yaml_dict["module"][self.name].keys()
+            }
+        elif name is not None:
+            self.name = name
+            self._control_fields = {}
+            self.yaml_dict = None
+
+        self._constants = {}
+
+        self.label = label
+
+        self.parameters = None
+        self.dsp_block: Optional[DspBlockType] = None  # pyright:ignore
+        self.stage_memory_string: str = ""
+        self.stage_memory_parameters: tuple | None = None
+
+        if len(self.i.edges) >= 1:
+            thread_crossings = []
+            for edge in inputs.edges:
+                thread_crossings.append(len(set(edge.crossings)))  # pyright: ignore checked above
+
+            if not all(x == thread_crossings[0] for x in thread_crossings):
+                input_msg = "\n"
+
+                for i, edge in enumerate(self.i.edges):
+                    crossings_set = set(edge.crossings)  # pyright: ignore checked above
+                    if not crossings_set:
+                        input_msg += f"Input {i} does not cross any threads.\n"
+                    else:
+                        input_msg += f"Input {i} crosses threads {crossings_set}.\n"  # pyright: ignore checked above
+
+                raise RuntimeError(
+                    f"\nAll stage inputs to {type(self).__name__} (label={self.label})"
+                    " must cross the same number of threads.\n"
+                    f"Currently, inputs cross {thread_crossings} threads.\nInputs with less than "
+                    f"{max(thread_crossings)} thread crossings must pass through Stages on"
+                    " earlier threads to avoid a latency mismatch and thread blocking.\n"
+                    "A Bypass Stage can be added on an earlier thread if no additional DSP is needed."
+                    + input_msg
+                )
+
+            self.crossings = list(set(self.i.edges[0].crossings))  # pyright: ignore checked above
+
+    def __init_subclass__(cls) -> None:
+        """Add all subclasses of Stage to a global list for querying."""
+        super().__init_subclass__()
+        _GlobalStages.stages.append(cls)
+
+    def set_parameters(self, parameters: StageParameterType):  # pyright:ignore
+        """Use a pydantic model to update the runtime parameters of a Stage."""
+        if isinstance(parameters, StageParameters) and type(parameters) != StageParameters:
+            raise NotImplementedError(
+                f"A subclass of StageParameters ({type(parameters).__name__}) "
+                "was passed to the generic implementation, of set_parameters, resulting in the "
+                "parameters not being used. Please define set_parameters for the specific Stage class."
+            )
+        # This is a generic implementation, so it does nothing.
+        # Subclasses should override this method to set the parameters
+        pass
+
+    @property
+    def o(self) -> StageOutputList:
+        """
+        This stage's outputs. Use this object to connect this stage to the next stage in the pipeline.
+        Subclass must call self.create_outputs() for this to exist.
+        """
+        if self._o is None:
+            raise RuntimeError("Stage must add outputs with create_outputs in its __init__ method")
+        return self._o
+
+    def create_outputs(self, n_out):
+        """
+        Create this stages outputs.
+
+        Parameters
+        ----------
+        n_out : int
+            number of outputs to create.
+        """
+        self.n_out = n_out
+        o = []
+        for i in range(n_out):
+            output = StageOutput(fs=self.fs, frame_size=self.frame_size, crossings=self.crossings)
+            output.source_index = i
+            output.set_source(self)
+            o.append(output)
+        self._o = StageOutputList(o)
+
+    def __setitem__(self, key, value):
+        """Support for dictionary like access to config fields."""
+        if key not in self._control_fields:
+            raise KeyError(
+                f"{key} is not a valid control field for {self.name}, try one of {', '.join(self._control_fields.keys())}"
+            )
+        self._control_fields[key].value = value
+
+    def __getitem__(self, key):
+        """Support for dictionary like access to config fields."""
+        if key not in self._control_fields:
+            raise KeyError(
+                f"{key} is not a valid control field for {self.name}, try one of {', '.join(self._control_fields.keys())}"
+            )
+        return self._control_fields[key].value
+
+    def set_control_field_cb(self, field, getter, setter=None):
+        """
+        Register callbacks for getting and setting control fields, to be called by classes which implement stage.
+
+        Parameters
+        ----------
+        field : str
+            name of the field
+        getter : function
+            A function which returns the current value
+        setter : function
+            A function which accepts 1 argument that will be used as the new value
+        """
+        if field not in self._control_fields:
+            raise KeyError(
+                f"{field} is not a valid control field for {self.name}, try one of {', '.join(self._control_fields.keys())}"
+            )
+
+        self._control_fields[field] = PropertyControlField(getter, setter)
+
+    def set_constant(self, field, value, value_type):
+        """
+        Define constant values in the stage. These will be hard coded in
+        the autogenerated code and cannot be changed at runtime.
+
+        Parameters
+        ----------
+        field : str
+            name of the field
+        value : ndarray or int or float or list
+            value of the constant. This can be an array or scalar
+
+        """
+        if not isinstance(value, (int, float, numpy.ndarray, list)):
+            raise TypeError(f"Type {type(value)} not a supported Stage constant value format")
+
+        if isinstance(value, numpy.ndarray) and value.ndim > 1:
+            raise TypeError(f"Only 1D numpy arrays can be set as Stage constants")
+
+        self._constants[field] = value
+
+    @property
+    def constants(self):
+        """Get a copy of the constants for this stage."""
+        # Copy so that the caller cannot modify
+        return {k: v for k, v in self._constants.items()}
+
+    def get_config(self):
+        """Get a dictionary containing the current value of the control
+        fields which have been set.
+
+        Returns
+        -------
+        dict
+            current control fields
+        """
+        ret = {}
+        for command_name, cf in self._control_fields.items():
+            if cf.value is not None:
+                ret[command_name] = cf.value
+        return ret
+
+    def process(self, in_channels):
+        """
+        Run dsp object on the input channels and return the output.
+
+        Args:
+            in_channels: list of numpy arrays
+
+        Returns
+        -------
+            list of numpy arrays.
+        """
+        # use float implementation as it is faster
+        return self.dsp_block.process_frame(in_channels)
+
+    def get_frequency_response(self, nfft=32768) -> tuple[numpy.ndarray, numpy.ndarray]:
+        """
+        Return the frequency response of this instance's dsp_block attribute.
+
+        Parameters
+        ----------
+        nfft
+            The length of the FFT
+
+        Returns
+        -------
+        ndarray, ndarray
+            Frequency values, Frequency response for this stage.
+        """
+        if self.dsp_block is None:
+            raise RuntimeError("This stage has not set its dsp_block")
+        return self.dsp_block.freq_response(nfft)
+
+    def plot_frequency_response(self, nfft=32768):
+        """
+        Plot magnitude and phase response of this stage using matplotlib. Will
+        be displayed inline in a jupyter notebook.
+
+        Parameters
+        ----------
+        nfft : int
+            Number of frequency bins to calculate in the fft.
+        """
+        f, h = self.get_frequency_response(nfft)
+        plot.plot_frequency_response(f, h, name=self.name)
+
+    def add_to_dot(self, dot):
+        """
+        Add this stage to a diagram that is being constructed.
+        Does not add the edges.
+
+        Parameters
+        ----------
+        dot : graphviz.Diagraph
+            dot instance to add edges to.
+        """
+        inputs = "|".join(f"<i{i}> " for i in range(self.n_in))
+        outputs = "|".join(f"<o{i}> " for i in range(self.n_out))
+        center = f"{self.index}: {type(self).__name__}\\n"
+
+        def render_details_r(value, indent="", first=True):
+            """Recursively process the details of a stage."""
+            next_indent = indent + "  "
+            new_line = "\\n" if not first else ""
+            if isinstance(value, dict):
+                return new_line + f"\\n{indent}".join(
+                    f"{k}: {render_details_r(v, next_indent, False)}" for k, v in value.items()
+                )
+            if isinstance(value, (list, tuple)):
+                return new_line + f"\\n{indent}".join(
+                    f"{i}: {render_details_r(v, next_indent, True)}" for i, v in enumerate(value)
+                )
+            return str(value)
+
+        def render_details(value):
+            """
+            Belt and braces - escapes the chars which need escaping.
+
+            As per https://graphviz.org/doc/info/shapes.html#record.
+            """
+            details = render_details_r(value)
+            # replaces the chars [ ] < > | { } with \[ \] \< \> \| \{ \}
+            return re.sub(r"(\||<|>|\[|\]|\{|\})", r"\\\1", details)
+
+        if self.label:
+            center = f"{self.index}: {self.label}\\n"
+        if self.parameters:
+            details = render_details(self.parameters.model_dump())
+            label = f"{{ {{ {inputs} }} | {center} | {details} | {{ {outputs} }}}}"
+        else:
+            label = f"{{ {{ {inputs} }} | {center} | {{ {outputs} }}}}"
+
+        dot.node(self.id.hex, label)
+
+    def get_required_allocator_size(self):
+        """
+        Calculate the required statically-allocated memory in bytes for this stage.
+        Formats this into a compile-time determinable expression.
+
+        Returns
+        -------
+            compile-time determinable expression of required allocator size.
+        """
+        macro_name = f"{self.name.upper()}_STAGE_REQUIRED_MEMORY"
+        if self.stage_memory_parameters is not None:
+            return f"{macro_name}({','.join((str(x) for x in self.stage_memory_parameters))})"
+        else:
+            return macro_name
+
+
+def all_stages() -> dict[str, Type[Stage]]:
+    """Get a dict containing all stages in scope."""
+    return {s.__name__: s for s in _GlobalStages.stages}

lib_audio_dsp/python/audio_dsp/design/templates/cmd_offsets_h.mako

@@ -0,0 +1,44 @@
+// Copyright 2024 XMOS LIMITED.
+// This Software is subject to the terms of the XMOS Public Licence: Version 1.
+#ifndef CMD_OFFSETS_H
+#define CMD_OFFSETS_H
+
+#include <stages/adsp_module.h>
+#include <stddef.h>
+
+#include "cmds.h"
+%for name in cmd_map:
+#include "${name}_config.h"
+%endfor
+
+typedef struct {
+    uint32_t cmd_id; // CmdID
+    uint32_t offset;    // offset
+    uint32_t size;      //size
+}module_config_offsets_t;
+
+%for name, data in cmd_map.items():
+// Offset and size of fields in the ${name}_config_t structure
+static module_config_offsets_t ${name}_config_offsets[] = {
+%for field_name, field_data in data.items():
+<% field_data["size"] = field_data["size"] if "size" in field_data else 1 %>\
+{.cmd_id=CMD_${name.upper()}_${field_name.upper()}, .offset=offsetof(${name}_config_t, ${field_name}), .size=sizeof(${field_data["type"]}) * ${field_data["size"]}},
+%endfor
+};
+%endfor
+
+static module_config_offsets_t *ptr_module_offsets[] = {
+%for name, data in cmd_map.items():
+    ${name}_config_offsets,
+%endfor
+};
+
+typedef enum
+{
+%for name in cmd_map:
+    e_dsp_stage_${name},
+%endfor
+    num_dsp_stages
+}all_dsp_stages_t;
+
+#endif

lib_audio_dsp/python/audio_dsp/design/templates/cmds_h.mako

@@ -0,0 +1,17 @@
+// Copyright 2024 XMOS LIMITED.
+// This Software is subject to the terms of the XMOS Public Licence: Version 1.
+#ifndef CMDS_H
+#define CMDS_H
+
+%for name, data in cmd_map.items():
+// ${name} module commands
+<% cmd_id = 1 %>\
+%for field_name, field_data in data.items():
+#define CMD_${name.upper()}_${field_name.upper()}             ${cmd_id}
+<% cmd_id = cmd_id + 1 %>\
+%endfor
+#define NUM_CMDS_${name.upper()}    ${len(data)}    // Number of commands in the ${name} module
+
+%endfor
+
+#endif

lib_audio_dsp/python/audio_dsp/design/templates/gen_command_map_offsets_c.mako

@@ -0,0 +1,47 @@
+// Copyright 2024 XMOS LIMITED.
+// This Software is subject to the terms of the XMOS Public Licence: Version 1.
+
+#include <stdio.h>
+
+%for name in cmd_map:
+#include "${name}_config.h"
+%endfor
+
+#if OFFSET_GEN
+int main(int argc, char* argv[])
+{
+    FILE *fp;
+    printf("argc = %d\n", argc);
+    if(argc == 2)
+    {
+        printf("argv = %s\n", argv[1]);
+        fp = fopen(argv[1], "w");
+    }
+    else
+    {
+        fp = fopen("cmd_offset_size.h", "w");
+    }
+    fprintf(fp, "#ifndef CMD_NUM_VALUES_H\n");
+    fprintf(fp, "#define CMD_NUM_VALUES_H\n\n");
+
+    // Number of values read or written as part of the command. Used in the host cmd_map.c file. Needs to be generated by running
+    // an application since the config yaml files might specified size in terms of #defines (for eg. FILTERS * NUM_COEFFS_PER_BIQUAD)
+%for name, data in cmd_map.items():
+    %for field_name, field_data in data.items():
+<%
+    if "size" in field_data:
+        size = field_data["size"]
+    elif field_data["type"] == "float_s32_t":
+        size = 2
+    else:
+        size = 1
+%>\
+    fprintf(fp, "#define NUM_VALUES_${name.upper()}_${field_name.upper()} %u\n", ${size});
+    fprintf(fp, "\n");
+    %endfor
+%endfor
+    fprintf(fp, "#endif\n");
+    fclose(fp);
+    return 0;
+}
+#endif

lib_audio_dsp/python/audio_dsp/design/templates/host_cmd_map.mako

@@ -0,0 +1,18 @@
+// Copyright 2024 XMOS LIMITED.
+// This Software is subject to the terms of the XMOS Public Licence: Version 1.
+#include "cmd_num_values.h"
+#include "cmds.h"
+
+static cmd_t commands[] =
+{
+%for name, data in cmd_map.items():
+    %for field_name, field_data in data.items():
+<%
+    field_data["size"] = field_data["size"] if "size" in field_data else 1
+    cmd_type = {'int32_t': 'TYPE_INT32', 'int16_t': 'TYPE_INT16', 'int8_t': 'TYPE_CHAR', 'uint8_t': 'TYPE_UINT8', 'float': 'TYPE_FLOAT', 'int': 'TYPE_INT32', 'uint32_t': 'TYPE_UINT32', 'float_s32_t': 'TYPE_INT32', 'int32_t*': 'TYPE_INT32'}
+%>\
+ {0xff, "${name.upper()}_${field_name.upper()}", ${cmd_type[field_data["type"]]}, CMD_${name.upper()}_${field_name.upper()}, CMD_READ_WRITE, NUM_VALUES_${name.upper()}_${field_name.upper()}, "${field_data["help"].strip()}", false},
+    %endfor
+
+%endfor
+};

lib_audio_dsp/python/audio_dsp/design/templates/py_stage_doc.mako

@@ -0,0 +1,135 @@
+.. _${title}_stages:
+
+${"#"*(len(title) + 7)}
+${title} Stages
+${"#"*(len(title) + 7)}
+
+${docstring}
+
+% for cl in classes:
+.. _${cl}_stage:
+
+${"="*len(cl)}
+${cl}
+${"="*len(cl)}
+
+.. autoclass:: audio_dsp.stages.${cl}
+    :noindex:
+    :members:
+    :inherited-members: Stage
+<%
+
+    ## Get the parameter type from the stage.set_parameters method
+    import importlib
+    import inspect
+    from typing import get_type_hints
+    import types
+    import audio_dsp
+    # Import the stage class
+    stages_mod = importlib.import_module('audio_dsp.stages')
+    stage_cls = getattr(stages_mod, cl)
+    # Get the set_parameters method
+    set_params = getattr(stage_cls, 'set_parameters')
+    # Get the type hints for the method
+    hints = get_type_hints(set_params)
+    # Assume the first argument after 'self' is the model class
+    params = inspect.signature(set_params).parameters
+    param_names = list(params.keys())
+    # Skip 'self', get the next parameter
+    param_name = param_names[1]
+    model_cls = hints.get(param_name)
+
+    if model_cls == audio_dsp.design.stage.StageParameterType:
+        # generic StageParameterType, so no specific model class
+        model_cls = []
+    elif type(model_cls) == types.UnionType:
+        # If it's a Union, there's multiple types
+        model_cls = list(model_cls.__args__)
+    else:
+        model_cls = [model_cls]
+
+    for i, cls in enumerate(model_cls):
+        # get the full class path
+        model_cls[i] = f"{cls.__module__}.{cls.__name__}"
+
+%>
+% for param_cls in model_cls:
+.. autopydantic_model:: ${param_cls}
+    :noindex:
+    :members:
+    :inherited-members: StageParameters
+% endfor ## for cls in model_cls
+
+
+${cl} Control
+${"="*len(cl)}========
+
+% if class_data[cl]:
+The following runtime command ids are available for the ${cl} Stage. For
+details on reading and writing these commands, see the Run-Time Control User Guide.
+
+<%
+  row_list = []
+  max_cmd = len("Command ID macro") + 2 # add 2 so the title doesn't ever get split in the PDF
+  max_pay = len("Payload length")
+  max_help = len("Description")
+  for field_name, field_data in class_data[cl].items():
+    import re
+    size_str = "*[" + str(field_data['size']) + "]" if "size" in field_data else ""
+    help_str = f'{field_data["help"].strip()}' if "help" in field_data else ""
+    if "rw_type" in field_data and field_data["rw_type"] == "CMD_READ_ONLY":
+      help_str += " This command is read only. When sending a write control command, it will be ignored."
+    safe_name = cl.replace("RMS", "Rms")
+    snake_name = re.sub(r'(?<!^)(?=[A-Z])', '_', safe_name).upper()
+    cmd_str = f"CMD_{snake_name}_{field_name.upper()}"
+    payload_str = f"``sizeof({field_data['type']}){size_str}``"
+    this_row = [cmd_str, payload_str, help_str]
+    row_list.append(this_row)
+  
+    if len(cmd_str) > max_cmd:
+      max_cmd = len(cmd_str)
+    if len(payload_str) > max_pay:
+      max_pay = len(payload_str)
+    if len(help_str) > max_help:
+      max_help = len(help_str)
+  
+  page_width = 74 # this is a guesstimate of number of chars in a page
+  cmd_width = int((max_cmd/page_width)*100)
+  pay_width = int(((max_pay - 5)/page_width)*100) # subtract ` chars and compensate for font (ish)
+  help_width = 100 - pay_width
+
+%>
+##  do the printing, use ljust to pad to max size
+.. table::
+  :widths: ${help_width}, ${pay_width}
+% if len(row_list) > 6:
+  :class: longtable
+%endif
+
+  ${"="*max_help}  ${"="*max_pay}
+  ${"Control parameter".ljust(max_help)}  ${"Payload length".ljust(max_pay)}
+  ${"="*max_help}  ${"="*max_pay}
+% for row in row_list:
+  ${row[0].ljust(max_help)}  ${row[1].ljust(max_pay)}
+% if "²" in row[2]:  ## ljust in Mako ignores ², no idea why
+  ${row[2].ljust(max_help + 1)}  ${'\\'.ljust(max_pay)}
+% else:
+  ${row[2].ljust(max_help)}  ${'\\'.ljust(max_pay)}
+%endif  ## "²" in row[2]:
+% if row_list.index(row) < len(row_list) - 1:
+  ## don't print a blank row at the end
+  |
+  ${"-"*max_help}--${"-"*max_pay}
+% endif  ## row_list.index(row) < len(row_list) - 1
+% endfor  ## row in row_list
+  ${"="*max_help}  ${"="*max_pay}
+
+% else:  ## class_data[cl]
+
+The ${cl} Stage has no runtime controllable parameters.
+
+% endif ## class_data[cl]
+
+% endfor ## cl in classes
+
+

lib_audio_dsp/python/audio_dsp/design/templates/struct_def_h.mako

@@ -0,0 +1,46 @@
+// Copyright 2024 XMOS LIMITED.
+// This Software is subject to the terms of the XMOS Public Licence: Version 1.
+#ifndef ${name.upper()}_CONFIG_H
+#define ${name.upper()}_CONFIG_H
+
+%for i in includes:
+#include <${i}>
+%endfor
+
+%for d in defines:
+#define ${d}    ${defines[d]}
+%endfor
+
+/** 
+ * This structure allows for real time control of the ${name} stage.
+ * It should be passed to the ${name}_control function.
+ */
+typedef struct
+{
+%for field_name, field_data in data.items():
+<%
+    attrib_str = f'{field_data["attribute"]} ' if "attribute" in field_data else ""
+    size_str = "[" + str(field_data['size']) + "]" if "size" in field_data else ""
+    help_str = f'{field_data["help"]} ' if "help" in field_data else ""
+%>\
+<%block filter="wrap_helpstr">
+    ${help_str}
+</%block>
+    ${field_data["type"]} ${attrib_str}${field_name}${size_str};
+
+%endfor
+}${name}_config_t;
+
+#endif
+
+<%!
+# This is a function to wrap long help descriptions
+import textwrap
+def wrap_helpstr(text):
+    line_len = 80
+    max_len = line_len - 4 - 4 - 3
+    if len(text) > line_len:
+        return "    /**\n     * " + '\n'.join(textwrap.wrap(text.strip(), line_len, subsequent_indent='     * ')).strip() + "\n     */"
+    else:
+        return f"    /** {text.strip()} */"
+%>

lib_audio_dsp/python/audio_dsp/design/thread.py

@@ -0,0 +1,88 @@
+# Copyright 2024-2025 XMOS LIMITED.
+# This Software is subject to the terms of the XMOS Public Licence: Version 1.
+"""Contains classes for adding a thread to the DSP pipeline."""
+
+from typing import Type
+
+from audio_dsp.design.composite_stage import CompositeStage, _StageOrComposite
+from audio_dsp.design.stage import Stage, StageOutputList, find_config
+
+
+class DSPThreadStage(Stage):
+    """
+    Stage for the DSP thread. Does not support processing of data through it. Only
+    used for DSP thread level control commands, for example, querying the max cycles
+    consumed by the thread.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(config=find_config("dsp_thread"), **kwargs)
+        self.create_outputs(0)
+
+    def add_to_dot(self, dot):
+        """
+        Exclude this stage from the dot diagram.
+
+        Parameters
+        ----------
+        dot : graphviz.Diagraph
+            dot instance to add edges to.
+        """
+
+
+class Thread(CompositeStage):
+    """
+    A composite stage used to represent a thread in the pipeline. Create
+    using Pipeline.thread rather than instantiating directly.
+
+    Parameters
+    ----------
+    id : int
+        Thread index
+    kwargs : dict
+        forwarded to __init__ of CompositeStage
+
+    Attributes
+    ----------
+    id : int
+        Thread index
+    thread_stage : Stage
+        DSPThreadStage stage
+    """
+
+    def __init__(self, id: int, **kwargs):
+        super().__init__(name=f"Thread {id}", **kwargs)
+        self.id = id
+        self.thread_stage = None
+
+    def __enter__(self):
+        """Support for context manager."""
+        return self
+
+    def __exit__(self, type, value, traceback):
+        """Support for context manager."""
+        ...
+
+    def add_thread_stage(self):
+        """Add to this thread the stage which manages thread level commands."""
+        self.thread_stage = self.stage(DSPThreadStage, StageOutputList(), label=f"thread{self.id}")
+
+    def stage(
+        self,
+        stage_type: Type[_StageOrComposite],
+        inputs: StageOutputList,
+        label: str | None = None,
+        **kwargs,
+    ) -> _StageOrComposite:
+        """
+        Create a new stage or composite stage and
+        register it with this composite stage.
+
+        This is a wrapper around :func:`audio_dsp.design.composite_stage.stage`
+        but adds a thread crossing counter.
+        """
+        for i in inputs.edges:
+            if i:
+                i.crossings.append(self.id)
+        stage = super().stage(stage_type, inputs, label, **kwargs)
+        return stage