This commit is contained in:
Steven Dan
2025-12-11 09:43:42 +08:00
commit d8b2974133
1822 changed files with 280037 additions and 0 deletions

View File

@@ -0,0 +1,194 @@
# Copyright 2024-2025 XMOS LIMITED.
# This Software is subject to the terms of the XMOS Public Licence: Version 1.
"""Collection of tuning utilities for the pipeline."""
from audio_dsp.design.pipeline import Pipeline
from audio_dsp.tuning.transport import *
from typing import Literal
import numpy as np
import tabulate
def _validate_pipeline_checksum(pipeline: Pipeline, proto: TuningTransport):
"""
Check if Python and device pipeline checksums match.
Raise a runtime error if the checksums are not equal.
Assumes that proto is an already-connected TuningTransport.
Parameters
----------
pipeline : Python pipeline for which to validate checksum against the device pipeline
"""
assert pipeline.pipeline_stage is not None # To stop ruff from complaining
payload = CommandPayload(pipeline.pipeline_stage, "checksum", None)
device_pipeline_checksum = proto.read(payload)
equal = np.array_equal(
np.array(device_pipeline_checksum),
np.array(pipeline.pipeline_stage["checksum"]),
)
if equal is False:
raise RuntimeError(
(
"Device pipeline mismatch; the pipeline on the connected device does not match "
"this design. To resolve this, update the firmware on the connected device to use this updated pipeline.\n"
f"\n\tExpected checksum: {pipeline.pipeline_stage['checksum']}\n\tGot {device_pipeline_checksum}"
)
)
def send_config_to_device(pipeline: Pipeline, protocol: TuningTransport):
"""
Send the current config for all stages to the device.
Make sure set_host_app() is called before calling this to set a valid host app.
Parameters
----------
pipeline : Pipeline
A designed and optionally tuned pipeline
protocol : TuningTransport
An initialised subclass of TuningTransport to use for communicating with the device
"""
with protocol as proto:
_validate_pipeline_checksum(pipeline, proto)
for stage in pipeline.stages:
for command, value in stage.get_config().items():
payload = CommandPayload(stage, command, value)
proto.write(payload)
_ValidationMode = Literal["once"] | Literal["never"] | Literal["always"]
class ConfigSender:
"""
A stateful config sender which has a memory of the last configs that were sent to
the device and only sends configs which have changed.
Example usage::
from audio_dsp.tuning.transport.xscope import XScopeTransport
from audio_dsp.tuning import ConfigSender
from audio_dsp.design.pipeline import Pipeline
p = Pipeline(generate_xscope_task=True)
t = XScopeTransport()
c = config_sender(t)
# Send config to the device
c(p)
Parameters
----------
protocol: TuningTransport
An initialised subclass of TuningTransport to use for communicating with the
device.
validation_mode: str
One of "always", "never", or "once", determines how ofter the pipeline checksum is validated.
"""
def __init__(self, protocol: TuningTransport, validation_mode: _ValidationMode = "always"):
self._protocol = protocol
self._last_config = {}
self._validation_mode = validation_mode
if self._validation_mode not in {"once", "always", "never"}:
raise ValueError(f"Invalid validation mode {validation_mode}")
self._validated = False
def __call__(self, pipeline: Pipeline) -> None:
"""
Send updated pipeline parameters.
Parameters
----------
pipeline:
The pipeline to send to the device.
"""
commands = {}
for stage in pipeline.stages:
stage_index = stage.index
for command, value in stage.get_config().items():
c = (stage_index, command, value)
commands[str(c)] = c
with self._protocol as proto:
if self._validation_mode == "always" or (
self._validation_mode == "once" and not self._validated
):
_validate_pipeline_checksum(pipeline, self._protocol)
self._validated = True
for command_str in (c for c in commands if c not in self._last_config):
command = commands[command_str]
payload = CommandPayload(pipeline.stages[command[0]], command[1], command[2])
proto.write(payload)
self._last_config = commands
def profile_pipeline(pipeline: Pipeline, protocol: TuningTransport):
"""
Profiles the DSP threads that are a part of the pipeline.
Parameters
----------
pipeline : Pipeline
A designed and optionally tuned pipeline
"""
with protocol as proto:
_validate_pipeline_checksum(pipeline, proto)
# print("Thread Index Max Cycles")
profile_info = []
for thread in pipeline.threads:
thread_fs = None
thread_frame_size = None
stages = thread.get_all_stages()
for stg in stages:
if stg.fs is not None:
thread_fs = stg.fs
thread_frame_size = stg.frame_size
break
# Assuming that all stages in the thread have the same sampling freq and frame size
if thread_fs is None:
raise RuntimeError(
f"Could not find out the sampling frequency for thread index {thread.id}"
)
if thread_frame_size is None:
raise RuntimeError(
f"Could not find out the frame size for thread index {thread.id}"
)
reference_timer_freq_hz = 100e6
frame_time_s = float(thread_frame_size) / thread_fs
ticks_per_sample_time_s = reference_timer_freq_hz * frame_time_s
ticks_per_sample_time_s = ticks_per_sample_time_s
# TODO Implement a generic way of reading all config from the stage
payload = CommandPayload(thread.thread_stage, "max_cycles", None)
cycles = proto.read(payload)
percentage_used = (cycles / ticks_per_sample_time_s) * 100
profile_info.append(
[
thread.id,
round(ticks_per_sample_time_s, 2),
cycles,
round(percentage_used, 2),
]
)
print(
tabulate.tabulate(
profile_info,
headers=[
"thread index",
"available time (ref timer ticks)",
"max ticks consumed",
"% consumed",
],
tablefmt="pretty",
)
)

View File

@@ -0,0 +1,209 @@
# Copyright 2024-2025 XMOS LIMITED.
# This Software is subject to the terms of the XMOS Public Licence: Version 1.
"""This module defines the base classes and types for DSP tuning transport mechanisms."""
import abc
import contextlib
import numpy as np
import struct
import types
from audio_dsp.design.stage import Stage
ValType = int | float | str | np.integer
MultiValType = ValType | list[ValType] | tuple[ValType, ...] | None
class CommandPayload:
"""Class for holding all relevant information regarding a command."""
cmd_types_byte_lengths = {
"uint8_t": 1,
"int8_t": 1,
"int16_t": 2,
"int": 4,
"int32_t": 4,
"uint32_t": 4,
"int32_t*": 4,
"float": 4,
"float_s32_t": 4,
}
cmd_types_struct_map = {
"uint8_t": "B",
"int8_t": "b",
"int16_t": "h",
"int": "i",
"int32_t": "i",
"uint32_t": "I",
"int32_t*": "I",
"float": "f",
"float_s32_t": "f",
}
def __init__(self, stage: Stage, command: str, value: MultiValType) -> None:
assert stage.index is not None
assert stage.yaml_dict is not None
stage_index = stage.index
name = stage.name
module_yaml: dict[str, dict] = stage.yaml_dict["module"][name]
try:
cmd_index = list(module_yaml.keys()).index(command) + 1
cmd_type = module_yaml[command]["type"]
cmd_n_values = module_yaml[command].get("size", 1)
except ValueError as e:
print(f"Command {command} not valid for stage {name}")
raise e from None
self.stage_index: int = stage_index
self.values: MultiValType = value
self.cmd_id: int = cmd_index
self.cmd_n_values: int = cmd_n_values
self.cmd_type: str = cmd_type
self.stage: Stage = stage
self.command: str = command
def to_bytes(self) -> tuple[int, bytes | None]:
"""Convert this commands' values into a set of raw bytes."""
retnum = 0
retvals = None
try:
cmd_n_bytes = self.cmd_n_values * self.cmd_types_byte_lengths[self.cmd_type]
except IndexError as e:
print(f"Command type {self.cmd_type} size unknown, please add to class")
raise e from None
retnum = cmd_n_bytes
if self.values is not None:
if isinstance(self.values, (int, float, str, np.integer)):
# Single argument
retvals = self._transform_single_value(self.values)
retnum = len(retvals)
elif isinstance(self.values, (list, tuple)):
# Multiple arguments
concat = bytes()
for elem in self.values:
concat += self._transform_single_value(elem)
retvals = concat
retnum = len(retvals)
else:
# ???
print(f"Transformation of values of type {type(self.values)} unknown.")
raise DevicePayloadError
if retnum != cmd_n_bytes:
print(
f"Length error: {retnum} != {cmd_n_bytes} for {self.stage_index}:{self.cmd_id} with value {self.values}"
)
raise DevicePayloadError
return (retnum, retvals)
def _transform_single_value(self, value: ValType) -> bytes:
if isinstance(value, int):
return self._transform_int(value)
elif isinstance(value, float):
return self._transform_float(value)
elif isinstance(value, np.integer):
return self._transform_npint(value)
else: # string
if "." in value:
# treat strings as floats if they have a . in them.
# there are error cases https://stackoverflow.com/a/20929881
return self._transform_float(float(value))
else:
return self._transform_int(int(value))
def _transform_int(self, value: int) -> bytes:
if self.cmd_type in ("float", "float_s32_t"):
raise DevicePayloadError
return value.to_bytes(self.cmd_n_values, "big")
def _transform_float(self, value: float) -> bytes:
if self.cmd_type not in ("float", "float_s32_t"):
raise DevicePayloadError
return bytes(struct.pack("!f", value))
def _transform_npint(self, value: np.integer) -> bytes:
if self.cmd_type in ("float", "float_s32_t"):
raise DevicePayloadError
return value.tobytes()
def from_bytes(self, data: bytes) -> "CommandPayload":
"""Convert bytes received from the device on issue of this command to
bytes and return a valid CommandPayload object.
"""
struct_code = f"{self.cmd_n_values}{self.cmd_types_struct_map[self.cmd_type]}"
ret: MultiValType = struct.unpack(struct_code, data)
if len(ret) == 1:
ret = ret[0]
return CommandPayload(self.stage, self.command, ret)
class DeviceConnectionError(Exception):
"""Raised when the tuning transport classes cannot connect to the device."""
pass
class DevicePayloadError(Exception):
"""Raised when the payload specified to a transport class is malformed."""
class TuningTransport(contextlib.AbstractContextManager["TuningTransport"], abc.ABC):
"""Base class for different transport media for tuning commands."""
def __enter__(self) -> "TuningTransport":
"""
Call subclass' connect() method. This ensures that the subclass is
instantiated correctly when used as a context manager.
"""
return self.connect()
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: types.TracebackType | None,
) -> bool | None:
"""
Call the subclass' disconnect() method. Ensures that the subclass
disconnects cleanly when used as a context manager, including when
exceptions occur.
"""
return self.disconnect()
@abc.abstractmethod
def connect(self) -> "TuningTransport":
"""
Perform any required operations to set up a connection to the device
and make the device ready to receive control commands.
"""
raise NotImplementedError
@abc.abstractmethod
def write(self, payload: CommandPayload) -> int:
"""Send a command to the device."""
raise NotImplementedError
@abc.abstractmethod
def read(self, payload: CommandPayload) -> MultiValType:
"""
Read data from the device. This is expected to perform a write operation
with no payload to request the device make ready the requested data,
then the device is expected to transmit the requested data back to the host.
"""
raise NotImplementedError
@abc.abstractmethod
def disconnect(self):
"""
Perform any required operations to cleanly shut down the interface with
the device.
"""
raise NotImplementedError
from .xscope import XScopeTransport

View File

@@ -0,0 +1,145 @@
# Copyright 2024-2025 XMOS LIMITED.
# This Software is subject to the terms of the XMOS Public Licence: Version 1.
"""This module implements the XScopeTransport class for managing DSP tuning
communication over xscope, including endpoint handling and command type mappings.
"""
from . import (
CommandPayload,
TuningTransport,
DeviceConnectionError,
MultiValType,
)
from .xscope_endpoint import Endpoint, QueueConsumer
class SilentEndpoint(Endpoint):
"""Subclass of Endpoint which silences the on_register callback.
Consequently, this subclass does not generate any print() statements
in normal operation.
"""
def on_register(self, id_, type_, name, unit, data_type):
"""Handle server probe registration events. In this case, do nothing."""
return
class XScopeTransport(TuningTransport):
"""
Manages all methods required to communicate tuning over xscope.
Parameters
----------
hostname : str
Hostname of the xscope server to which to attempt a connection.
Defaults to 'localhost'.
port : str
Port of the xscope server to which to attempt a connection.
Defaults to '12345'.
probe_name : str
Name of the xscope probe over which to receive data from the device.
Defaults to 'ADSP'.
"""
def __init__(
self, hostname: str = "localhost", port: str = "12345", probe_name="ADSP"
) -> None:
self.ep = SilentEndpoint()
self.hostname = hostname
self.port = port
self.connected = False
self.read_queue = QueueConsumer(self.ep, probe_name)
def connect(self) -> "XScopeTransport":
"""
Make a connection to a running xscope server.
Returns
-------
self : XScopeTransport
If this function returns, this object is guaranteed to be connected
to a running xscope server
Raises
------
DeviceConnectionError
If connection to the xscope server at {self.hostname}:{self.port} fails.
"""
if not self.connected:
ret = self.ep.connect(self.hostname, self.port)
if ret == 0:
self.connected = True
else:
raise DeviceConnectionError
return self
def write(self, payload: CommandPayload, read_cmd: bool = False) -> int:
"""
Assemble a valid packet of bytes to send to the device via the connected
xscope server, and then send them. Sets the top bit of the command ID if
this is sending a read command.
Parameters
----------
payload : CommandPayload
The command to write.
read_cmd : bool
Whether the command to be sent is a read command (True) or a write command (False).
Returns
-------
int
Return code: 0 if success, 1 if failure
Raises
------
DeviceConnectionError
If this instance is not connected to a device. Call .connect().
DevicePayloadError
If the stated size of the payload in payload.size is not equal to
the number of bytes that the value in payload.value is represented
by when cast to the type in payload.cmd_type.
"""
if not self.connected:
raise DeviceConnectionError
# Target schema is instance_id, cmd_id, payload_len, payload
command_id = payload.cmd_id | (0x80 if read_cmd else 0x00)
n_bytes, values = payload.to_bytes()
payload_bytes = bytes([payload.stage_index, command_id, n_bytes])
if values is not None:
payload_bytes += values
# print(f"sent: {payload_bytes}")
return self.ep.publish(payload_bytes)
def read(self, payload: CommandPayload) -> MultiValType:
"""
Send a read command to the device over xscope, and then wait for the reply.
Parameters
----------
payload : CommandPayload
The command to write.
Returns
-------
ValType | tuple[ValType, ...]
Tuple of data received from the device. The device sends raw bytes;
this function automatically casts the received data to the type
specified in payload.cmd_type.
"""
self.write(payload, read_cmd=True)
data = self.read_queue.next()
# We know that for this specific application, data will always be bytes
assert isinstance(data, bytes)
return payload.from_bytes(data).values
def disconnect(self):
"""Shut down the connection to the currently connected xscope server."""
self.ep.disconnect()
self.connected = False

View File

@@ -0,0 +1,367 @@
# Copyright 2024-2025 XMOS LIMITED.
# This Software is subject to the terms of the XMOS Public Licence: Version 1.
"""Python bindings for Tools-supplied xscope_endpoint.dll/.so.
Adapted and updated from XMOS_TOOLS_DIR/lib/python/xscope.py.
"""
import os, ctypes, platform, sys, time, queue
from collections import defaultdict
import ctypes.util
import pathlib
import typing
import numpy as np
"""
Function prototypes to match the c functions defined in xscope_endpoint.h
"""
PRINT_CALLBACK = ctypes.CFUNCTYPE(
None,
ctypes.c_ulonglong, # timestamp
ctypes.c_uint, # length
ctypes.c_char_p,
) # data
RECORD_CALLBACK = ctypes.CFUNCTYPE(
None,
ctypes.c_uint, # id
ctypes.c_ulonglong, # timestamp
ctypes.c_uint, # length
ctypes.c_ulonglong, # dataval
ctypes.POINTER(ctypes.c_char),
) # databytes
REGISTER_CALLBACK = ctypes.CFUNCTYPE(
None,
ctypes.c_uint, # id
ctypes.c_uint, # type
ctypes.c_uint, # r
ctypes.c_uint, # g
ctypes.c_uint, # b
ctypes.c_char_p, # name
ctypes.c_char_p, # unit
ctypes.c_uint, # data_type
ctypes.c_char_p,
) # data_name
STATS_CALLBACK = ctypes.CFUNCTYPE(
None,
ctypes.c_uint, # id
ctypes.c_ulonglong,
) # average
ProbeInfo = typing.NamedTuple(
"ProbeInfo",
[
("type", int),
("r", int),
("g", int),
("b", int),
("name", str),
("unit", str),
("data_type", int),
("data_name", str),
],
)
ConsumerPrototype = typing.Callable[[int, str, int, int, bytes], None]
class Endpoint:
"""Python xSCOPE endpoint wrapper.
Example:
def test_callback(timestamp, probe_name, length, data_val, data_bytes):
print(
"{} {} {} {} {}".format(timestamp, probe_name, length, data_val, data_bytes)
)
ep = Endpoint()
try:
if ep.connect('localhost', '12345'):
print "Failed to connect"
else:
ep.consume(callback, 'my_probe")
while(True):
pass
except KeyboardInterrupt:
ep.disconnect()
"""
def __init__(self):
self._probe_info: dict[int, ProbeInfo] = {} # probe id to probe info lookup.
# probe_info includes name, units, data type, etc...
self._consumers: typing.DefaultDict[str, set[ConsumerPrototype]] = defaultdict(
set
) # probe name -> callbacks lookup
# NOTE: The consumers must be looked up by name and not id because
# they can be specified before the probe_info is defined
tool_path = os.environ.get("XMOS_TOOL_PATH")
assert tool_path is not None
tool_path = pathlib.Path(tool_path)
ps = platform.system()
if ps == "Windows":
lib_path = tool_path / "lib" / "xscope_endpoint.dll"
else: # Darwin (aka MacOS) or Linux
lib_path = tool_path / "lib" / "xscope_endpoint.so"
self.lib_xscope = ctypes.CDLL(str(lib_path))
# create callbacks
print_cb = self._print_callback_func()
self.lib_xscope.xscope_ep_set_print_cb(print_cb)
record_cb = self._record_callback_func()
self.lib_xscope.xscope_ep_set_record_cb(record_cb)
register_cb = self._register_callback_func()
self.lib_xscope.xscope_ep_set_register_cb(register_cb)
stats_cb = self._stats_callback_func()
self.lib_xscope.xscope_ep_set_stats_cb(stats_cb)
def _print_callback_func(self):
def func(timestamp: int, length: int, data: bytes):
self.on_print(timestamp, data[0:length])
return PRINT_CALLBACK(func)
def _register_callback_func(self):
def func(
id_: int,
type_: int,
r: int,
g: int,
b: int,
name: bytes,
unit: bytes,
data_type: int,
data_name: bytes,
):
pi = ProbeInfo(
type_,
r,
g,
b,
name.decode(),
unit.decode(),
data_type,
data_name.decode(),
)
self._probe_info[id_] = pi
self.on_register(id_, type_, name, unit, data_type)
return REGISTER_CALLBACK(func)
def _record_callback_func(self):
def func(id_: int, timestamp: int, length: int, data_val: int, data_bytes: bytes):
self.on_record(id_, timestamp, length, data_val, data_bytes)
return RECORD_CALLBACK(func)
def _stats_callback_func(self):
def func(id_: int, average: int):
# TODO
pass
return STATS_CALLBACK(func)
def on_print(self, timestamp: int, data: bytes):
"""XScope printf handler.
Override this to method to implement your own printing or to silence the printout.
"""
print("DEV: " + data.decode().rstrip())
def on_register(self, id_: int, type_: int, name: bytes, unit: bytes, data_type: int):
"""Server probe registration handler.
Override this to method to implement your own registration or to silence the printout.
"""
print(f"Probe registered: {id_=}, {type_=}, {name=}, {unit=}, {data_type=}")
def on_record(self, id_: int, timestamp: int, length: int, data_val: int, data_bytes: bytes):
"""Server record handler. Will dispatch to probe consumer callback.
Override this to method to implement your own dispatcher. However,
that should rarely be necessary.
"""
# For some reason negative values keep getting interpreted as UINT64
# which messes everything up. This seems to fix it.
data_val = int(np.int32(np.uint32(data_val & 0xFFFFFFFF)))
def notify_consumers(consumers: set[ConsumerPrototype], probe_name: str):
for cb in consumers:
cb(timestamp, probe_name, length, data_val, data_bytes)
probe_info = self._probe_info[id_]
probe_name = probe_info.name
if probe_name in self._consumers:
notify_consumers(self._consumers[probe_name], probe_name)
if "*" in self._consumers:
notify_consumers(self._consumers["*"], probe_name)
def connect(self, hostname: str = "localhost", port: str = "12345") -> int:
"""Connect to xSCOPE server.
Args:
hostname (str): Hostname of running xSCOPE server.
port (str): Port number of running xSCOPE server.
Returns
-------
0 for success
1 for failure
"""
return self.lib_xscope.xscope_ep_connect(hostname.encode(), port.encode())
def disconnect(self):
"""Disconnect from xSCOPE server."""
self.lib_xscope.xscope_ep_disconnect()
def consume(self, callback: ConsumerPrototype, probe_name: str | None = None):
"""Consume a probe by name.
Note:
Set probe=None to consume all probes.
Args:
callback (callable): Callback function with signature (timestamp, probe_name, length, data_val, data_bytes).
probe_name (str): Probe name
"""
probe_name = "*" if probe_name is None else probe_name
self._consumers[probe_name].add(callback)
def publish(self, data: bytes) -> int:
"""Publish message to endpoint.
The length of data must be <= 255.
Args:
data: Bytes to publish.
Returns
-------
0 for success
1 for failure
"""
return self.lib_xscope.xscope_ep_request_upload(
ctypes.c_uint(len(data) + 1), ctypes.c_char_p(data)
)
class QueueConsumer:
"""
Attaches to a specified Endpoint class and probe. Automatically consumes all
data from the probe and places it in a queue. The queue may then be queried
with the .next() method.
Parameters
----------
ep : Endpoint
Endpoint object to which to attach this consumer
probe : str
Name of the probe for which to queue data. Set to None to consume all probes.
probe_timeout : float
"""
def __init__(
self,
ep: Endpoint,
probe: str,
/,
probe_timeout: float = 10.0,
mode: typing.Literal["values", "bytes"] = "bytes",
):
self.ep = ep
self.probe_timeout = probe_timeout
self.ep.consume(self._consume, probe)
self.queue = queue.Queue()
self.mode = mode
def _consume(
self,
timestamp: int,
probe_name: str,
length: int,
data_val: int,
data_bytes: bytes,
):
if self.mode == "values":
self.queue.put(data_val)
elif self.mode == "bytes":
self.queue.put(data_bytes[:length])
def next(self, count: int = 1) -> int | bytes | list[int] | list[bytes]:
"""
Get the next item from the queue.
Parameters
----------
count : int
Number of items from the queue to return. Defaults to 1.
Raises
------
Queue.Empty
The probes timed out when trying to get values from xscope.
"""
# At least in some scenarios, ctrl-c fails to interrupt the script and
# just hangs forever if it's blocking on queue.get(). The timeout is
# currently serving as a watchdog so that the pytest process doesn't need
# to be killed through extraordinary means.
if count == 1:
return self.queue.get(timeout=self.probe_timeout)
r = []
for _ in range(count):
r.append(self.queue.get(timeout=self.probe_timeout))
return r
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser("Python xSCOPE")
parser.add_argument("--host", default="localhost", help="Hostname")
parser.add_argument("--port", default="12345", help="Port")
parser.add_argument(
"-c",
"--consume",
nargs="?",
action="append",
default=[],
help="Probe names to consume (omit to consume all)",
)
parser.add_argument("-p", "--publish", default=None, help="Message to publish")
args = parser.parse_args()
def test_callback(timestamp, probe_name, length, data_val, data_bytes):
"""Prints received values."""
print("{} {} {} {} {}".format(timestamp, probe_name, length, data_val, data_bytes))
ep = Endpoint()
try:
if ep.connect(args.host, args.port):
print("Failed to connect")
sys.exit(1)
if args.publish:
ep.publish(args.publish)
if args.consume:
for probe in args.consume:
ep.consume(test_callback, probe)
else:
ep.consume(test_callback)
while True:
# Release the CPU
time.sleep(1)
except KeyboardInterrupt:
ep.disconnect()