From d35d4ab773847f2ef1d82eb1bd46e276da4d507e Mon Sep 17 00:00:00 2001 From: JWahle Date: Mon, 14 Aug 2023 20:28:54 +0200 Subject: [PATCH 01/37] Changed format of /api/storedconfigs and /api/storedcoeffs from returning arrays with filenames and last modified dates to returning just an array with {name, lastModified} --- backend/filemanagement.py | 30 +++++++++++++++++++----------- backend/views.py | 6 +++--- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/backend/filemanagement.py b/backend/filemanagement.py index d33dbe1..3459c77 100644 --- a/backend/filemanagement.py +++ b/backend/filemanagement.py @@ -2,7 +2,7 @@ import os import zipfile from copy import deepcopy -from os.path import isfile, islink, split, join, realpath, relpath, normpath, isabs, commonpath, getmtime +from os.path import isfile, split, join, realpath, relpath, normpath, isabs, commonpath, getmtime import logging import traceback @@ -57,16 +57,24 @@ def list_of_files_in_directory(folder): """ files = [file_in_folder(folder, file) for file in os.listdir(folder) - if os.path.isfile(file_in_folder(folder, file))] - files_list = [] - for f in files: - fname = os.path.basename(f) - filetime = getmtime(f) - files_list.append((fname, filetime)) - sorted_files = sorted(files_list, key=lambda x: x[0].lower()) - sorted_names = [file[0] for file in sorted_files] - sorted_times = [file[1] for file in sorted_files] - return (sorted_names, sorted_times) + if isfile(file_in_folder(folder, file))] + files_list = map( + lambda file: { + "name": (os.path.basename(file)), + "lastModified": (getmtime(file)) + }, + files + ) + sorted_files = sorted(files_list, key=lambda x: x["name"].lower()) + return sorted_files + + +def list_of_filenames_in_directory(folder): + return map( + lambda file: file["name"], + list_of_files_in_directory(folder) + ) + def delete_files(folder, files): diff --git a/backend/views.py b/backend/views.py index 072ffa6..66d2f82 100644 --- a/backend/views.py +++ b/backend/views.py @@ -13,7 +13,7 @@ zip_response, zip_of_files, get_yaml_as_json, set_as_active_config, get_active_config, save_config, make_config_filter_paths_absolute, coeff_dir_relative_to_config_dir, replace_relative_filter_path_with_absolute_paths, make_config_filter_paths_relative, - make_absolute, replace_tokens_in_filter_config + make_absolute, replace_tokens_in_filter_config, list_of_filenames_in_directory ) from .filters import defaults_for_filter, filter_options, pipeline_step_options from .settings import get_gui_config_or_defaults @@ -186,7 +186,7 @@ async def eval_filter_values(request): replace_relative_filter_path_with_absolute_paths(config, config_dir) channels = content["channels"] samplerate = content["samplerate"] - filter_file_names, _ = list_of_files_in_directory(request.app["coeff_dir"]) + filter_file_names = list_of_filenames_in_directory(request.app["coeff_dir"]) if "filename" in config["parameters"]: filename = config["parameters"]["filename"] options = filter_options(filter_file_names, filename) @@ -221,7 +221,7 @@ async def eval_filterstep_values(request): config["devices"]["samplerate"] = samplerate config["devices"]["capture"]["channels"] = channels plot_config = make_config_filter_paths_absolute(config, config_dir) - filter_file_names, _ = list_of_files_in_directory(request.app["coeff_dir"]) + filter_file_names = list_of_filenames_in_directory(request.app["coeff_dir"]) options = pipeline_step_options(filter_file_names, config, step_index) for _, filt in plot_config.get("filters", {}).items(): replace_tokens_in_filter_config(filt, samplerate, channels) From f650b5f4e3ec2757fc66d5e7dfb60da47a8b0f49 Mon Sep 17 00:00:00 2001 From: JWahle Date: Mon, 21 Aug 2023 21:24:31 +0200 Subject: [PATCH 02/37] Implemented Convolver config import --- .github/workflows/build.yml | 2 +- backend/convolver_config_import.py | 167 ++++++++++++++++++ backend/convolver_config_import_test.py | 225 ++++++++++++++++++++++++ backend/filters_test.py | 13 +- backend/routes.py | 2 + backend/views.py | 14 +- 6 files changed, 412 insertions(+), 11 deletions(-) create mode 100644 backend/convolver_config_import.py create mode 100644 backend/convolver_config_import_test.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5805bce..47f0143 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: with: python-version: '3.12' - name: Run python tests - run: python3 -m unittest discover -p "*_test.py" + run: cd backend ; python3 -m unittest discover -p "*_test.py" - name: Install template render dependencies run: | diff --git a/backend/convolver_config_import.py b/backend/convolver_config_import.py new file mode 100644 index 0000000..6c42f55 --- /dev/null +++ b/backend/convolver_config_import.py @@ -0,0 +1,167 @@ +from os.path import split + + +def filename_of_path(path: str) -> str: + return split(path.replace('\\', '/'))[-1] + + +def channels_factors_and_inversions_as_list(channels_and_factors: str) -> list[tuple[int, float, bool]]: + channels_and_fractions = [channel_and_fraction.split('.') + for channel_and_fraction + in channels_and_factors.split(' ')] + return [ + (abs(int(channel)), + float('0.' + fraction), + channel[0] == '-') + for (channel, fraction) in channels_and_fractions] + + +class Filter: + filename: str + channel: int + channel_in_file: int + input_channels: list[tuple[int, float, bool]] + output_channels: list[tuple[int, float, bool]] + + def __init__(self, channel, filter_text: list[str]): + self.channel = channel + self.filename = filename_of_path(filter_text[0]) + self.channel_in_file = int(filter_text[1]) + self.input_channels = channels_factors_and_inversions_as_list(filter_text[2]) + self.output_channels = channels_factors_and_inversions_as_list(filter_text[3]) + + def name(self) -> str: + return self.filename + '-' + str(self.channel_in_file) + + +class ConvolverConfig: + + _samplerate: int + _input_channels: int + _output_channels: int + _input_delays: list[int] + _output_delays: list[int] + _filters: list[Filter] + + def __init__(self, config_text: str): + """ + :param config_text: a convolver config (https://convolver.sourceforge.net/config.html) as string + """ + lines = config_text.splitlines() + first_line_items = lines[0].split() + self._samplerate = int(first_line_items[0]) + self._input_channels = int(first_line_items[1]) + self._output_channels = int(first_line_items[2]) + self._input_delays = [int(x) for x in lines[1].split()] + self._output_delays = [int(x) for x in lines[2].split()] + filter_lines = lines[3:len(lines)] + filter_count = int(len(filter_lines) / 4) + self._filters = [Filter(n, filter_lines[n*4:n*4+4]) for n in range(filter_count)] + + def as_json(self) -> dict: + return { + 'devices': {'samplerate': self._samplerate}, + 'filters': self._delay_filter_definitions() | self._convolution_filter_definitions(), + 'mixers': self._mixer_in() | self._mixer_out(), + 'pipeline': + self._input_delay_pipeline_steps() + + self._mixer_in_pipeline_step() + + self._filter_pipeline_steps() + + self._mixer_out_pipeline_step() + + self._output_delay_pipeline_steps() + } + + def _delay_filter_definitions(self) -> dict: + delays = set(self._input_delays + self._output_delays) + delays.remove(0) + return {self._delay_name(delay): self._delay_filter(delay) for delay in delays} + + @staticmethod + def _delay_name(delay: int) -> str: + return 'Delay'+str(delay) + + @staticmethod + def _delay_filter(delay: int) -> dict: + return { + 'type': 'Delay', + 'parameters': {'delay': delay, 'unit': 'ms', 'subsample': False} + } + + def _convolution_filter_definitions(self) -> dict: + return { + f.name(): { + 'type': 'Conv', + 'parameters': {'type': 'Wav', 'filename': f.filename, 'channel': f.channel_in_file} + } + for f in self._filters + } + + def _input_delay_pipeline_steps(self) -> list[dict]: + return self._delay_pipeline_steps(self._input_delays) + + def _delay_pipeline_steps(self, delays: list[int]) -> list[dict]: + return [ + { + 'type': 'Filter', + 'channel': channel, + 'names': [self._delay_name(delay)], + 'bypassed': None, + 'description': None, + } + for channel, delay in enumerate(delays) if delay != 0 + ] + + def _output_delay_pipeline_steps(self) -> list[dict]: + return self._delay_pipeline_steps(self._output_delays) + + def _mixer_in(self) -> dict: + return { + 'Mixer in': { + 'channels': { + 'in': self._input_channels, + 'out': max(1, len(self._filters)) + }, + 'mapping': [ + {'dest': f.channel, + 'sources': [ + {'channel': channel, 'gain': factor, 'scale': 'linear', 'inverted': invert} + for (channel, factor, invert) in f.input_channels + ]} + for f in self._filters + ] + } + } + + def _mixer_out(self) -> dict: + return { + 'Mixer out': { + 'channels': { + 'in': max(1, len(self._filters)), + 'out': self._output_channels + }, + 'mapping': [ + {'dest': output_channel, + 'sources': [ + {'channel': f.channel, 'gain': factor, 'scale': 'linear', 'inverted': invert} + for f in self._filters + for (channel, factor, invert) in f.output_channels + if channel == output_channel + ]} + for output_channel in range(self._output_channels) + ] + } + } + + @staticmethod + def _mixer_in_pipeline_step() -> list[dict]: + return [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}] + + @staticmethod + def _mixer_out_pipeline_step() -> list[dict]: + return [{'type': 'Mixer', 'name': 'Mixer out', 'description': None}] + + def _filter_pipeline_steps(self) -> list[dict]: + return [ + {'type': 'Filter', 'channel': f.channel, 'names': [f.name()], 'bypassed': None, 'description': None} + for f in self._filters + ] diff --git a/backend/convolver_config_import_test.py b/backend/convolver_config_import_test.py new file mode 100644 index 0000000..1f43fba --- /dev/null +++ b/backend/convolver_config_import_test.py @@ -0,0 +1,225 @@ +from unittest import TestCase +from textwrap import dedent +from convolver_config_import import ConvolverConfig, filename_of_path, channels_factors_and_inversions_as_list + + +def clean_multi_line_string(multiline_text: str): + """ + :param multiline_text: + :return: the text without the first blank line and indentation + """ + return dedent(multiline_text.removeprefix('\n')) + + +class Test(TestCase): + + def test_filename_of_path(self): + self.assertEqual('File.wav', filename_of_path('File.wav')) + self.assertEqual('File.wav', filename_of_path('/some/path/File.wav')) + self.assertEqual('File.wav', filename_of_path('C:\\some\\path\\File.wav')) + + def test_channels_factors_and_inversions_as_list(self): + self.assertEqual( + channels_factors_and_inversions_as_list("0.0 1.1 -9.9 -0.0"), + [(0, 0.0, False), (1, 0.1, False), (9, 0.9, True), (0, 0, True)] + ) + + def test_samplerate_is_imported(self): + convolver_config = clean_multi_line_string(""" + 96000 1 2 0 + 0 + 0 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual(json['devices'], {'samplerate': 96000}) + + def test_delays_and_mixers_are_imported(self): + convolver_config = clean_multi_line_string(""" + 96000 2 3 0 + 3 + 0 4 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual( + json['filters'], + {'Delay3': {'type': 'Delay', 'parameters': {'delay': 3, 'unit': 'ms', 'subsample': False}}, + 'Delay4': {'type': 'Delay', 'parameters': {'delay': 4, 'unit': 'ms', 'subsample': False}}} + ) + self.assertEqual( + json['mixers']['Mixer in']['channels'], + {'in': 2, 'out': 1} + ) + self.assertEqual( + json['mixers']['Mixer out']['channels'], + {'in': 1, 'out': 3} + ) + self.assertEqual( + json['pipeline'], + [{'type': 'Filter', 'channel': 0, 'names': ['Delay3'], 'bypassed': None, 'description': None}, + {'type': 'Mixer', 'name': 'Mixer in', 'description': None}, + {'type': 'Mixer', 'name': 'Mixer out', 'description': None}, + {'type': 'Filter', 'channel': 1, 'names': ['Delay4'], 'bypassed': None, 'description': None}] + ) + + def test_simple_impulse_response(self): + convolver_config = clean_multi_line_string(""" + 0 1 1 0 + 0 + 0 + IR.wav + 0 + 0.0 + 0.0 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual( + json['filters'], + {'IR.wav-0': { + 'type': 'Conv', + 'parameters': {'type': 'Wav', 'filename': 'IR.wav', 'channel': 0}}} + ) + self.assertEqual( + json['pipeline'], + [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}, + {'type': 'Filter', 'channel': 0, 'names': ['IR.wav-0'], 'bypassed': None, 'description': None}, + {'type': 'Mixer', 'name': 'Mixer out', 'description': None}] + ) + + def test_path_is_ignored_for_impulse_response_files(self): + convolver_config = clean_multi_line_string(""" + 0 1 1 0 + 0 + 0 + IR1.wav + 0 + 0.0 + 0.0 + C:\\any/path/IR2.wav + 0 + 0.0 + 0.0 + /some/other/path/IR3.wav + 0 + 0.0 + 0.0 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual(json['filters']['IR1.wav-0']['parameters']['filename'], 'IR1.wav') + self.assertEqual(json['filters']['IR2.wav-0']['parameters']['filename'], 'IR2.wav') + self.assertEqual(json['filters']['IR3.wav-0']['parameters']['filename'], 'IR3.wav') + + def test_wav_file_with_multiple_impulse_responses(self): + convolver_config = clean_multi_line_string(""" + 0 1 1 0 + 0 + 0 + IR.wav + 0 + 0.0 + 0.0 + IR.wav + 1 + 0.0 + 0.0 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual(json['filters']['IR.wav-0']['parameters']['channel'], 0) + self.assertEqual(json['filters']['IR.wav-1']['parameters']['channel'], 1) + + def test_impulse_responses_are_mapped_to_correct_channels(self): + convolver_config = clean_multi_line_string(""" + 0 1 1 0 + 0 + 0 + IR1.wav + 0 + 0.0 + 0.0 + IR2.wav + 0 + 0.0 + 0.0 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual( + json['pipeline'], + [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}, + {'type': 'Filter', 'channel': 0, 'names': ['IR1.wav-0'], 'bypassed': None, 'description': None}, + {'type': 'Filter', 'channel': 1, 'names': ['IR2.wav-0'], 'bypassed': None, 'description': None}, + {'type': 'Mixer', 'name': 'Mixer out', 'description': None}] + ) + + def test_impulse_response_with_input_scaling(self): + convolver_config = clean_multi_line_string(""" + 0 2 2 0 + 0 0 + 0 0 + IR.wav + 0 + 0.0 1.1 + 0.0 + IR.wav + 1 + 0.2 1.3 + 0.0 + IR.wav + 2 + -1.5 -0.4 + 0.0 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual( + json['mixers']['Mixer in'], + {'channels': {'in': 2, 'out': 3}, + 'mapping': [ + {'dest': 0, + 'sources': [ + {'channel': 0, 'gain': 0.0, 'scale': 'linear', 'inverted': False}, + {'channel': 1, 'gain': 0.1, 'scale': 'linear', 'inverted': False}]}, + {'dest': 1, + 'sources': [ + {'channel': 0, 'gain': 0.2, 'scale': 'linear', 'inverted': False}, + {'channel': 1, 'gain': 0.3, 'scale': 'linear', 'inverted': False}]}, + {'dest': 2, + 'sources': [ + {'channel': 1, 'gain': 0.5, 'scale': 'linear', 'inverted': True}, + {'channel': 0, 'gain': 0.4, 'scale': 'linear', 'inverted': True} + ]}, + ]} + ) + + def test_impulse_response_with_output_scaling(self): + convolver_config = clean_multi_line_string(""" + 0 2 2 0 + 0 0 + 0 0 + IR.wav + 0 + 0.0 + 0.0 1.1 + IR.wav + 1 + 0.0 + 0.2 1.3 + IR.wav + 2 + 0.0 + -1.5 -0.4 + """) + json = ConvolverConfig(convolver_config).as_json() + self.assertEqual( + json['mixers']['Mixer out'], + {'channels': {'in': 3, 'out': 2}, + 'mapping': [ + {'dest': 0, + 'sources': [ + {'channel': 0, 'gain': 0.0, 'scale': 'linear', 'inverted': False}, + {'channel': 1, 'gain': 0.2, 'scale': 'linear', 'inverted': False}, + {'channel': 2, 'gain': 0.4, 'scale': 'linear', 'inverted': True}]}, + {'dest': 1, + 'sources': [ + {'channel': 0, 'gain': 0.1, 'scale': 'linear', 'inverted': False}, + {'channel': 1, 'gain': 0.3, 'scale': 'linear', 'inverted': False}, + {'channel': 2, 'gain': 0.5, 'scale': 'linear', 'inverted': True}]}, + ]} + ) diff --git a/backend/filters_test.py b/backend/filters_test.py index cee7c56..2dbcceb 100644 --- a/backend/filters_test.py +++ b/backend/filters_test.py @@ -1,9 +1,8 @@ -import unittest +from unittest import TestCase +from filters import filter_options, pipeline_step_options -from backend.filters import filter_options, pipeline_step_options - -class FiltersTest(unittest.TestCase): +class FiltersTest(TestCase): def test_filter_options_with_samplerate(self): self.assertEqual( @@ -172,8 +171,4 @@ def test_pipeline_step_options_for_many_samplerate_and_channel_options(self): {"name": "48000 Hz - 2 Channels", "samplerate": 48000, "channels": 2}, {"name": "48000 Hz - 8 Channels", "samplerate": 48000, "channels": 8} ] - ) - - -if __name__ == '__main__': - unittest.main() + ) \ No newline at end of file diff --git a/backend/routes.py b/backend/routes.py index eb93c81..0f7efc9 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -12,6 +12,7 @@ set_active_config_name, config_to_yml, yml_to_json, + convolver_to_json, yml_config_to_json_config, validate_config, get_gui_index, @@ -50,6 +51,7 @@ def setup_routes(app): app.router.add_post("/api/configtoyml", config_to_yml) app.router.add_post("/api/ymlconfigtojsonconfig", yml_config_to_json_config) app.router.add_post("/api/ymltojson", yml_to_json) + app.router.add_post("/api/convolvertojson", convolver_to_json) app.router.add_post("/api/validateconfig", validate_config) app.router.add_get("/api/storedconfigs", get_stored_configs) app.router.add_get("/api/storedcoeffs", get_stored_coeffs) diff --git a/backend/views.py b/backend/views.py index 66d2f82..463cabe 100644 --- a/backend/views.py +++ b/backend/views.py @@ -17,6 +17,7 @@ ) from .filters import defaults_for_filter, filter_options, pipeline_step_options from .settings import get_gui_config_or_defaults +from .convolver_config_import import ConvolverConfig OFFLINE_CACHE = { "cdsp_status": "Offline", @@ -368,7 +369,7 @@ async def config_to_yml(request): async def yml_config_to_json_config(request): """ - Parse a yml string and return as json. + Parse a yml config string and return as json. """ config_ymlstr = await request.text() validator = request.app["VALIDATOR"] @@ -380,12 +381,22 @@ async def yml_config_to_json_config(request): async def yml_to_json(request): """ Parse a yml string and return as json. + This could also be just a partial config. """ yml = await request.text() loaded = yaml.safe_load(yml) return web.json_response(loaded) +async def convolver_to_json(request): + """ + Parse a Convolver config string and return as json. + """ + config = await request.text() + loaded = ConvolverConfig(config).as_json() + return web.json_response(loaded) + + async def validate_config(request): """ Validate a config, returned completed config. @@ -537,6 +548,7 @@ async def get_playback_devices(request): devs = cdsp.general.list_playback_devices(backend) return web.json_response(devs) + async def get_backends(request): """ Get lists of available playback and capture backends. From dddcd0a9d029f1eabc4f439333ba55e358f8987f Mon Sep 17 00:00:00 2001 From: JWahle Date: Sun, 14 Jan 2024 01:06:55 +0100 Subject: [PATCH 03/37] Added GUI configuration for bass and treble --- config/gui-config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config/gui-config.yml b/config/gui-config.yml index a0415a0..984a30b 100644 --- a/config/gui-config.yml +++ b/config/gui-config.yml @@ -6,3 +6,17 @@ hide_playback_device: false hide_rate_monitoring: false apply_config_automatically: false status_update_interval: 100 +custom_shortcuts: + - section: "Equalizer" + description: "To use the EQ, add filters named \"Bass\" and \"Treble\" to the pipeline.
Recommented settings:
Bass: Biquad Lowshelf freq=85 q=0.9
Treble: Biquad Highshelf freq=6500 q=0.7" + shortcuts: + - name: "Treble (dB)" + path_in_config: ["filters", "Treble", "parameters", "gain"] + range_from: -12 + range_to: 12 + step: 0.5 + - name: "Bass (dB)" + path_in_config: ["filters", "Bass", "parameters", "gain"] + range_from: -12 + range_to: 12 + step: 0.5 \ No newline at end of file From f28406040b4b2adb932af1697d63eb6c2d8b1e59 Mon Sep 17 00:00:00 2001 From: JWahle Date: Sun, 14 Jan 2024 01:59:29 +0100 Subject: [PATCH 04/37] Added documentation for custom shortcut configuration to README.md --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 5b232df..32e12b6 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,24 @@ supported_capture_types: ["Alsa", "File", "Stdin"] supported_playback_types: ["Alsa", "File", "Stdout"] ``` +### Adding custom shortcut settings +It is possible to configure custom shortcuts for the `Shortcuts` section and the compact view. +Here is an example config to set the gain of a filter called `MyFilter` within the range from 0 to 10 db in steps of 0.1 dB. +``` +custom_shortcuts: + - section: "My custom section" + description: "Optional description for the section. Omit the attribute, if unwanted" + shortcuts: + - name: "My filter gain" + description: "Optional description for the setting. Omit the attribute, if unwanted" + path_in_config: ["filters", "MyFilter", "parameters", "gain"] + range_from: 0 + range_to: 10 + step: 0.1 + - name: "The next setting" + ... +``` + ### Integrating with other software If you want to integrate CamillaGUI with other software, there are some options to customize the UI for your particular needs. From cb548a0832567d0871e142b6e6ff0c0c488122bc Mon Sep 17 00:00:00 2001 From: Henrik Date: Sat, 20 Jan 2024 20:51:51 +0100 Subject: [PATCH 05/37] Big renaming to avoid using misleading names --- backend/convolver_config_import.py | 2 +- backend/convolver_config_import_test.py | 46 ++++++++--------- backend/filemanagement.py | 24 ++++----- backend/filters.py | 10 ++-- backend/filters_test.py | 32 ++++++------ backend/routes.py | 12 ++--- backend/views.py | 69 +++++++++++++------------ 7 files changed, 98 insertions(+), 97 deletions(-) diff --git a/backend/convolver_config_import.py b/backend/convolver_config_import.py index 6c42f55..fd99204 100644 --- a/backend/convolver_config_import.py +++ b/backend/convolver_config_import.py @@ -58,7 +58,7 @@ def __init__(self, config_text: str): filter_count = int(len(filter_lines) / 4) self._filters = [Filter(n, filter_lines[n*4:n*4+4]) for n in range(filter_count)] - def as_json(self) -> dict: + def to_object(self) -> dict: return { 'devices': {'samplerate': self._samplerate}, 'filters': self._delay_filter_definitions() | self._convolution_filter_definitions(), diff --git a/backend/convolver_config_import_test.py b/backend/convolver_config_import_test.py index 1f43fba..22347c1 100644 --- a/backend/convolver_config_import_test.py +++ b/backend/convolver_config_import_test.py @@ -30,8 +30,8 @@ def test_samplerate_is_imported(self): 0 0 """) - json = ConvolverConfig(convolver_config).as_json() - self.assertEqual(json['devices'], {'samplerate': 96000}) + conf = ConvolverConfig(convolver_config).to_object() + self.assertEqual(conf['devices'], {'samplerate': 96000}) def test_delays_and_mixers_are_imported(self): convolver_config = clean_multi_line_string(""" @@ -39,22 +39,22 @@ def test_delays_and_mixers_are_imported(self): 3 0 4 """) - json = ConvolverConfig(convolver_config).as_json() + conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - json['filters'], + conf['filters'], {'Delay3': {'type': 'Delay', 'parameters': {'delay': 3, 'unit': 'ms', 'subsample': False}}, 'Delay4': {'type': 'Delay', 'parameters': {'delay': 4, 'unit': 'ms', 'subsample': False}}} ) self.assertEqual( - json['mixers']['Mixer in']['channels'], + conf['mixers']['Mixer in']['channels'], {'in': 2, 'out': 1} ) self.assertEqual( - json['mixers']['Mixer out']['channels'], + conf['mixers']['Mixer out']['channels'], {'in': 1, 'out': 3} ) self.assertEqual( - json['pipeline'], + conf['pipeline'], [{'type': 'Filter', 'channel': 0, 'names': ['Delay3'], 'bypassed': None, 'description': None}, {'type': 'Mixer', 'name': 'Mixer in', 'description': None}, {'type': 'Mixer', 'name': 'Mixer out', 'description': None}, @@ -71,15 +71,15 @@ def test_simple_impulse_response(self): 0.0 0.0 """) - json = ConvolverConfig(convolver_config).as_json() + conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - json['filters'], + conf['filters'], {'IR.wav-0': { 'type': 'Conv', 'parameters': {'type': 'Wav', 'filename': 'IR.wav', 'channel': 0}}} ) self.assertEqual( - json['pipeline'], + conf['pipeline'], [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}, {'type': 'Filter', 'channel': 0, 'names': ['IR.wav-0'], 'bypassed': None, 'description': None}, {'type': 'Mixer', 'name': 'Mixer out', 'description': None}] @@ -103,10 +103,10 @@ def test_path_is_ignored_for_impulse_response_files(self): 0.0 0.0 """) - json = ConvolverConfig(convolver_config).as_json() - self.assertEqual(json['filters']['IR1.wav-0']['parameters']['filename'], 'IR1.wav') - self.assertEqual(json['filters']['IR2.wav-0']['parameters']['filename'], 'IR2.wav') - self.assertEqual(json['filters']['IR3.wav-0']['parameters']['filename'], 'IR3.wav') + conf = ConvolverConfig(convolver_config).to_object() + self.assertEqual(conf['filters']['IR1.wav-0']['parameters']['filename'], 'IR1.wav') + self.assertEqual(conf['filters']['IR2.wav-0']['parameters']['filename'], 'IR2.wav') + self.assertEqual(conf['filters']['IR3.wav-0']['parameters']['filename'], 'IR3.wav') def test_wav_file_with_multiple_impulse_responses(self): convolver_config = clean_multi_line_string(""" @@ -122,9 +122,9 @@ def test_wav_file_with_multiple_impulse_responses(self): 0.0 0.0 """) - json = ConvolverConfig(convolver_config).as_json() - self.assertEqual(json['filters']['IR.wav-0']['parameters']['channel'], 0) - self.assertEqual(json['filters']['IR.wav-1']['parameters']['channel'], 1) + conf = ConvolverConfig(convolver_config).to_object() + self.assertEqual(conf['filters']['IR.wav-0']['parameters']['channel'], 0) + self.assertEqual(conf['filters']['IR.wav-1']['parameters']['channel'], 1) def test_impulse_responses_are_mapped_to_correct_channels(self): convolver_config = clean_multi_line_string(""" @@ -140,9 +140,9 @@ def test_impulse_responses_are_mapped_to_correct_channels(self): 0.0 0.0 """) - json = ConvolverConfig(convolver_config).as_json() + conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - json['pipeline'], + conf['pipeline'], [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}, {'type': 'Filter', 'channel': 0, 'names': ['IR1.wav-0'], 'bypassed': None, 'description': None}, {'type': 'Filter', 'channel': 1, 'names': ['IR2.wav-0'], 'bypassed': None, 'description': None}, @@ -167,9 +167,9 @@ def test_impulse_response_with_input_scaling(self): -1.5 -0.4 0.0 """) - json = ConvolverConfig(convolver_config).as_json() + conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - json['mixers']['Mixer in'], + conf['mixers']['Mixer in'], {'channels': {'in': 2, 'out': 3}, 'mapping': [ {'dest': 0, @@ -206,9 +206,9 @@ def test_impulse_response_with_output_scaling(self): 0.0 -1.5 -0.4 """) - json = ConvolverConfig(convolver_config).as_json() + conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - json['mixers']['Mixer out'], + conf['mixers']['Mixer out'], {'channels': {'in': 3, 'out': 2}, 'mapping': [ {'dest': 0, diff --git a/backend/filemanagement.py b/backend/filemanagement.py index 3459c77..2534433 100644 --- a/backend/filemanagement.py +++ b/backend/filemanagement.py @@ -111,16 +111,16 @@ def zip_of_files(folder, files): return zip_buffer.getvalue() -def get_yaml_as_json(request, path): +def read_yaml_from_path_to_object(request, path): """ - Read a yaml file, return the validated content as json. + Read a yaml file at the given path, return the validated content as a Python object. """ validator = request.app["VALIDATOR"] validator.validate_file(path) return validator.get_config() -def get_active_config(request): +def get_active_config_path(request): """ Get the active config filename. """ @@ -163,7 +163,7 @@ def get_active_config(request): return None -def set_as_active_config(request, filepath): +def set_path_as_active_config(request, filepath): """ Persistlently set the given config file path as the active config. """ @@ -254,12 +254,12 @@ def _read_statefile_config_path(statefile_path): logging.error(f"Details: {e}") return None -def save_config(config_name, json_config, request): +def save_config_to_yaml_file(config_name, config_object, request): """ Write a given config object to a yaml file. """ config_file = path_of_configfile(request, config_name) - yaml_config = yaml.dump(json_config).encode('utf-8') + yaml_config = yaml.dump(config_object).encode('utf-8') with open(config_file, "wb") as f: f.write(yaml_config) @@ -273,27 +273,27 @@ def coeff_dir_relative_to_config_dir(request): return coeff_dir_with_folder_separator_at_end -def make_config_filter_paths_absolute(json_config, config_dir): +def make_config_filter_paths_absolute(config_object, config_dir): """ Convert paths to coefficient files in a config from relative to absolute. """ conversion = lambda path, config_dir=config_dir: make_absolute(path, config_dir) - return convert_config_filter_paths(json_config, conversion) + return convert_config_filter_paths(config_object, conversion) -def make_config_filter_paths_relative(json_config, config_dir): +def make_config_filter_paths_relative(config_object, config_dir): """ Convert paths to coefficient files in a config from absolute to relative. """ conversion = lambda path, config_dir=config_dir: make_relative(path, config_dir) - return convert_config_filter_paths(json_config, conversion) + return convert_config_filter_paths(config_object, conversion) -def convert_config_filter_paths(json_config, conversion): +def convert_config_filter_paths(config_object, conversion): """ Apply a path conversion to all filter coefficient paths of a config. """ - config = deepcopy(json_config) + config = deepcopy(config_object) filters = config.get("filters") if filters is not None: for filter_name in filters: diff --git a/backend/filters.py b/backend/filters.py index 55659e6..7c22cd4 100644 --- a/backend/filters.py +++ b/backend/filters.py @@ -36,7 +36,7 @@ def defaults_for_filter(file_path): return {} -def filter_options(filter_file_names, filename): +def filter_plot_options(filter_file_names, filename): """ Get the different available options for samplerate and channels for a set of coeffient files. """ @@ -65,7 +65,7 @@ def pattern_from_filter_file_name(path): return re.compile(pattern) -def pipeline_step_options(filter_file_names, config, step_index): +def pipeline_step_plot_options(filter_file_names, config, step_index): """ Get the combined available samplerate and channels options for a filter step. """ @@ -74,7 +74,7 @@ def pipeline_step_options(filter_file_names, config, step_index): all_samplerate_and_channel_options = set_of_all_samplerate_and_channel_options(samplerates_and_channels_for_filter) samplerate_and_channel_options = set_of_samplerate_and_channel_options_available_for_all_filters( all_samplerate_and_channel_options, samplerates_and_channels_for_filter) - return options_as_json(samplerate_and_channel_options) + return plot_options_to_object(samplerate_and_channel_options) def map_of_samplerates_and_channels_per_filter(config, filter_file_names, step_index): @@ -91,7 +91,7 @@ def map_of_samplerates_and_channels_per_filter(config, filter_file_names, step_i if filter["type"] == "Conv" and parameters["type"] in {"Raw", "Wav"}: filename = parameters["filename"] samplerates_and_channels_for_filter[filter_name] = samplerate_and_channel_pairs_from_options( - filter_options(filter_file_names, filename), + filter_plot_options(filter_file_names, filename), default_samplerate, default_channels ) @@ -131,7 +131,7 @@ def set_of_samplerate_and_channel_options_available_for_all_filters(samplerate_a return options_available_for_all_filters -def options_as_json(samplerate_and_channel_options): +def plot_options_to_object(samplerate_and_channel_options): """ Convert samplerate/channel options to an object suitable for conversion to json. """ diff --git a/backend/filters_test.py b/backend/filters_test.py index 2dbcceb..c5486bb 100644 --- a/backend/filters_test.py +++ b/backend/filters_test.py @@ -1,12 +1,12 @@ from unittest import TestCase -from filters import filter_options, pipeline_step_options +from filters import filter_plot_options, pipeline_step_plot_options class FiltersTest(TestCase): - def test_filter_options_with_samplerate(self): + def test_filter_plot_options_with_samplerate(self): self.assertEqual( - filter_options( + filter_plot_options( ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], "filter_$samplerate$_2" ), @@ -16,9 +16,9 @@ def test_filter_options_with_samplerate(self): ] ) - def test_filter_options_with_channels(self): + def test_filter_plot_options_with_channels(self): self.assertEqual( - filter_options( + filter_plot_options( ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], "filter_44100_$channels$" ), @@ -28,9 +28,9 @@ def test_filter_options_with_channels(self): ] ) - def test_filter_options_with_samplerate_and_channels(self): + def test_filter_plot_options_with_samplerate_and_channels(self): self.assertEqual( - filter_options( + filter_plot_options( ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], "filter_$samplerate$_$channels$" ), @@ -42,7 +42,7 @@ def test_filter_options_with_samplerate_and_channels(self): ] ) self.assertEqual( - filter_options( + filter_plot_options( ["filter_2_44100", "filter_8_44100", "filter_2_48000", "filter_8_48000"], "filter_$channels$_$samplerate$" ), @@ -54,18 +54,18 @@ def test_filter_options_with_samplerate_and_channels(self): ] ) - def test_filter_options_without_samplerate_and_channels(self): + def test_filter_plot_options_without_samplerate_and_channels(self): self.assertEqual( - filter_options( + filter_plot_options( ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], "filter_44100_2" ), [{"name": "filter_44100_2"}] ) - def test_filter_options_handles_filenames_with_brackets(self): + def test_filter_plot_options_handles_filenames_with_brackets(self): self.assertEqual( - filter_options( + filter_plot_options( [ "filter_((44100)_(2))", "filter_((44100)_(8))", @@ -82,7 +82,7 @@ def test_filter_options_handles_filenames_with_brackets(self): ] ) - def test_pipeline_step_options_for_only_one_samplerate_and_channel_option(self): + def test_pipeline_step_plot_options_for_only_one_samplerate_and_channel_option(self): config = { "devices": { "samplerate": 44100, @@ -123,11 +123,11 @@ def test_pipeline_step_options_for_only_one_samplerate_and_channel_option(self): "filter-48000-8" ] self.assertEqual( - pipeline_step_options(filter_file_names, config, 0), + pipeline_step_plot_options(filter_file_names, config, 0), [{"name": "44100 Hz - 2 Channels", "samplerate": 44100, "channels": 2}] ) - def test_pipeline_step_options_for_many_samplerate_and_channel_options(self): + def test_pipeline_step_plot_options_for_many_samplerate_and_channel_options(self): config = { "devices": { "samplerate": 44100, @@ -164,7 +164,7 @@ def test_pipeline_step_options_for_many_samplerate_and_channel_options(self): "filter-48000-8" ] self.assertEqual( - pipeline_step_options(filter_file_names, config, 0), + pipeline_step_plot_options(filter_file_names, config, 0), [ {"name": "44100 Hz - 2 Channels", "samplerate": 44100, "channels": 2}, {"name": "44100 Hz - 8 Channels", "samplerate": 44100, "channels": 8}, diff --git a/backend/routes.py b/backend/routes.py index 0f7efc9..9e1228f 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -11,9 +11,9 @@ get_default_config_file, set_active_config_name, config_to_yml, - yml_to_json, - convolver_to_json, - yml_config_to_json_config, + yaml_to_json, + translate_convolver_to_json, + parse_and_validate_yml_config_to_json, validate_config, get_gui_index, get_stored_coeffs, @@ -49,9 +49,9 @@ def setup_routes(app): app.router.add_get("/api/getdefaultconfigfile", get_default_config_file) app.router.add_post("/api/setactiveconfigfile", set_active_config_name) app.router.add_post("/api/configtoyml", config_to_yml) - app.router.add_post("/api/ymlconfigtojsonconfig", yml_config_to_json_config) - app.router.add_post("/api/ymltojson", yml_to_json) - app.router.add_post("/api/convolvertojson", convolver_to_json) + app.router.add_post("/api/ymlconfigtojsonconfig", parse_and_validate_yml_config_to_json) + app.router.add_post("/api/ymltojson", yaml_to_json) + app.router.add_post("/api/convolvertojson", translate_convolver_to_json) app.router.add_post("/api/validateconfig", validate_config) app.router.add_get("/api/storedconfigs", get_stored_configs) app.router.add_get("/api/storedcoeffs", get_stored_coeffs) diff --git a/backend/views.py b/backend/views.py index 463cabe..621da5c 100644 --- a/backend/views.py +++ b/backend/views.py @@ -10,12 +10,12 @@ from .filemanagement import ( path_of_configfile, store_files, list_of_files_in_directory, delete_files, - zip_response, zip_of_files, get_yaml_as_json, set_as_active_config, get_active_config, save_config, + zip_response, zip_of_files, read_yaml_from_path_to_object, set_path_as_active_config, get_active_config_path, save_config_to_yaml_file, make_config_filter_paths_absolute, coeff_dir_relative_to_config_dir, replace_relative_filter_path_with_absolute_paths, make_config_filter_paths_relative, make_absolute, replace_tokens_in_filter_config, list_of_filenames_in_directory ) -from .filters import defaults_for_filter, filter_options, pipeline_step_options +from .filters import defaults_for_filter, filter_plot_options, pipeline_step_plot_options from .settings import get_gui_config_or_defaults from .convolver_config_import import ConvolverConfig @@ -190,7 +190,7 @@ async def eval_filter_values(request): filter_file_names = list_of_filenames_in_directory(request.app["coeff_dir"]) if "filename" in config["parameters"]: filename = config["parameters"]["filename"] - options = filter_options(filter_file_names, filename) + options = filter_plot_options(filter_file_names, filename) else: options = [] replace_tokens_in_filter_config(config, samplerate, channels) @@ -223,7 +223,7 @@ async def eval_filterstep_values(request): config["devices"]["capture"]["channels"] = channels plot_config = make_config_filter_paths_absolute(config, config_dir) filter_file_names = list_of_filenames_in_directory(request.app["coeff_dir"]) - options = pipeline_step_options(filter_file_names, config, step_index) + options = pipeline_step_plot_options(filter_file_names, config, step_index) for _, filt in plot_config.get("filters", {}).items(): replace_tokens_in_filter_config(filt, samplerate, channels) try: @@ -255,18 +255,18 @@ async def set_config(request): Apply a new config to CamillaDSP. """ json = await request.json() - json_config = json["config"] + config_object = json["config"] config_dir = request.app["config_dir"] cdsp = request.app["CAMILLA"] validator = request.app["VALIDATOR"] - json_config_with_absolute_filter_paths = make_config_filter_paths_absolute(json_config, config_dir) + config_object_with_absolute_filter_paths = make_config_filter_paths_absolute(config_object, config_dir) if cdsp.is_connected(): try: - cdsp.config.set_active(json_config_with_absolute_filter_paths) + cdsp.config.set_active(config_object_with_absolute_filter_paths) except CamillaError as e: raise web.HTTPInternalServerError(text=str(e)) else: - validator.validate_config(json_config_with_absolute_filter_paths) + validator.validate_config(config_object_with_absolute_filter_paths) errors = validator.get_errors() if len(errors) > 0: return web.json_response(data=errors) @@ -284,7 +284,7 @@ async def get_default_config_file(request): else: raise web.HTTPNotFound(text="No default config") try: - json_config = make_config_filter_paths_relative(get_yaml_as_json(request, config), config_dir) + config_object = make_config_filter_paths_relative(read_yaml_from_path_to_object(request, config), config_dir) except CamillaError as e: logging.error(f"Failed to get default config file, error: {e}") raise web.HTTPInternalServerError(text=str(e)) @@ -292,13 +292,13 @@ async def get_default_config_file(request): logging.error("Failed to get default config file") traceback.print_exc() raise web.HTTPInternalServerError(text=str(e)) - return web.json_response(json_config) + return web.json_response(config_object) async def get_active_config_file(request): """ Get the active config. If no config is active, return the default config. """ - active_config_path = get_active_config(request) + active_config_path = get_active_config_path(request) logging.debug(active_config_path) default_config_path = request.app["default_config"] config_dir = request.app["config_dir"] @@ -309,7 +309,7 @@ async def get_active_config_file(request): else: raise web.HTTPNotFound(text="No active or default config") try: - json_config = make_config_filter_paths_relative(get_yaml_as_json(request, config), config_dir) + config_object = make_config_filter_paths_relative(read_yaml_from_path_to_object(request, config), config_dir) except CamillaError as e: logging.error(f"Failed to get active config from CamillaDSP, error: {e}") raise web.HTTPInternalServerError(text=str(e)) @@ -318,10 +318,10 @@ async def get_active_config_file(request): traceback.print_exc() raise web.HTTPInternalServerError(text=str(e)) if active_config_path: - json = {"configFileName": active_config_path, "config": json_config} + data = {"configFileName": active_config_path, "config": config_object} else: - json = {"config": json_config} - return web.json_response(json) + data = {"config": config_object} + return web.json_response(data) async def set_active_config_name(request): @@ -331,7 +331,7 @@ async def set_active_config_name(request): json = await request.json() config_name = json["name"] config_file = path_of_configfile(request, config_name) - set_as_active_config(request, config_file) + set_path_as_active_config(request, config_file) return web.Response(text="OK") @@ -343,63 +343,64 @@ async def get_config_file(request): config_name = request.query["name"] config_file = path_of_configfile(request, config_name) try: - json_config = make_config_filter_paths_relative(get_yaml_as_json(request, config_file), config_dir) + config_object = make_config_filter_paths_relative(read_yaml_from_path_to_object(request, config_file), config_dir) except CamillaError as e: raise web.HTTPInternalServerError(text=str(e)) - return web.json_response(json_config) + return web.json_response(config_object) async def save_config_file(request): """ Save a config to a given filename. """ - json = await request.json() - save_config(json["filename"], json["config"], request) + content = await request.json() + save_config_to_yaml_file(content["filename"], content["config"], request) return web.Response(text="OK") async def config_to_yml(request): """ - Convert a json config to yml string (for saving to disk etc). + Convert a json config to yaml string (for saving to disk etc). """ content = await request.json() conf_yml = yaml.dump(content) return web.Response(text=conf_yml) -async def yml_config_to_json_config(request): +async def parse_and_validate_yml_config_to_json(request): """ - Parse a yml config string and return as json. + Parse a yaml config string and return serialized as json. """ - config_ymlstr = await request.text() + config_yaml = await request.text() validator = request.app["VALIDATOR"] - validator.validate_yamlstring(config_ymlstr) + validator.validate_yamlstring(config_yaml) config = validator.get_config() return web.json_response(config) -async def yml_to_json(request): +async def yaml_to_json(request): """ - Parse a yml string and return as json. + Parse a yaml string and return seralized as json. This could also be just a partial config. """ - yml = await request.text() - loaded = yaml.safe_load(yml) + config_yaml = await request.text() + loaded = yaml.safe_load(config_yaml) return web.json_response(loaded) -async def convolver_to_json(request): +async def translate_convolver_to_json(request): """ - Parse a Convolver config string and return as json. + Parse a Convolver config string and return + as a CamillaDSP config serialized as json. """ config = await request.text() - loaded = ConvolverConfig(config).as_json() - return web.json_response(loaded) + translated = ConvolverConfig(config).to_object() + return web.json_response(translated) async def validate_config(request): """ - Validate a config, returned completed config. + Validate a config, returned a list of errors or OK. """ config_dir = request.app["config_dir"] config = await request.json() From 5fd0fa7ced6f45d83042baceac78f5e93f017d57 Mon Sep 17 00:00:00 2001 From: Henrik Date: Sat, 20 Jan 2024 21:43:22 +0100 Subject: [PATCH 06/37] Fix parsing of gain, format code --- backend/convolver_config_import.py | 174 ++++++++----- backend/convolver_config_import_test.py | 331 +++++++++++++++++------- 2 files changed, 352 insertions(+), 153 deletions(-) diff --git a/backend/convolver_config_import.py b/backend/convolver_config_import.py index fd99204..788d560 100644 --- a/backend/convolver_config_import.py +++ b/backend/convolver_config_import.py @@ -1,19 +1,41 @@ -from os.path import split +from os.path import basename def filename_of_path(path: str) -> str: - return split(path.replace('\\', '/'))[-1] - - -def channels_factors_and_inversions_as_list(channels_and_factors: str) -> list[tuple[int, float, bool]]: - channels_and_fractions = [channel_and_fraction.split('.') - for channel_and_fraction - in channels_and_factors.split(' ')] + """ + Return just the filename from a full path. + Accepts both Windows paths such as C:\temp\file.wav + and Unix paths such as /tmp/file.wav + """ + return basename(path.replace("\\", "/")) + + +def fraction_to_gain(fraction: str) -> float: + if int(fraction) == 0: + # Special case, n.0 means channel n with a linear gain of 1.0 + return 1.0 + # n.mmm means channel n with a linear gain of 0.mmm + return float(f"0.{fraction}") + + +def parse_channel_and_fraction(channel: str, fraction: str) -> (int, float, bool): + int_channel = abs(int(channel)) + gain = fraction_to_gain(fraction) + inverted = channel.startswith("-") + return (abs(int_channel), gain, inverted) + + +def channels_factors_and_inversions_as_list( + channels_and_factors: str, +) -> list[tuple[int, float, bool]]: + channels_and_fractions = [ + channel_and_fraction.split(".") + for channel_and_fraction in channels_and_factors.split(" ") + ] return [ - (abs(int(channel)), - float('0.' + fraction), - channel[0] == '-') - for (channel, fraction) in channels_and_fractions] + parse_channel_and_fraction(channel, fraction) + for (channel, fraction) in channels_and_fractions + ] class Filter: @@ -31,11 +53,10 @@ def __init__(self, channel, filter_text: list[str]): self.output_channels = channels_factors_and_inversions_as_list(filter_text[3]) def name(self) -> str: - return self.filename + '-' + str(self.channel_in_file) + return self.filename + "-" + str(self.channel_in_file) class ConvolverConfig: - _samplerate: int _input_channels: int _output_channels: int @@ -54,21 +75,23 @@ def __init__(self, config_text: str): self._output_channels = int(first_line_items[2]) self._input_delays = [int(x) for x in lines[1].split()] self._output_delays = [int(x) for x in lines[2].split()] - filter_lines = lines[3:len(lines)] + filter_lines = lines[3 : len(lines)] filter_count = int(len(filter_lines) / 4) - self._filters = [Filter(n, filter_lines[n*4:n*4+4]) for n in range(filter_count)] + self._filters = [ + Filter(n, filter_lines[n * 4 : n * 4 + 4]) for n in range(filter_count) + ] def to_object(self) -> dict: return { - 'devices': {'samplerate': self._samplerate}, - 'filters': self._delay_filter_definitions() | self._convolution_filter_definitions(), - 'mixers': self._mixer_in() | self._mixer_out(), - 'pipeline': - self._input_delay_pipeline_steps() - + self._mixer_in_pipeline_step() - + self._filter_pipeline_steps() - + self._mixer_out_pipeline_step() - + self._output_delay_pipeline_steps() + "devices": {"samplerate": self._samplerate}, + "filters": self._delay_filter_definitions() + | self._convolution_filter_definitions(), + "mixers": self._mixer_in() | self._mixer_out(), + "pipeline": self._input_delay_pipeline_steps() + + self._mixer_in_pipeline_step() + + self._filter_pipeline_steps() + + self._mixer_out_pipeline_step() + + self._output_delay_pipeline_steps(), } def _delay_filter_definitions(self) -> dict: @@ -78,20 +101,24 @@ def _delay_filter_definitions(self) -> dict: @staticmethod def _delay_name(delay: int) -> str: - return 'Delay'+str(delay) + return "Delay" + str(delay) @staticmethod def _delay_filter(delay: int) -> dict: return { - 'type': 'Delay', - 'parameters': {'delay': delay, 'unit': 'ms', 'subsample': False} + "type": "Delay", + "parameters": {"delay": delay, "unit": "ms", "subsample": False}, } def _convolution_filter_definitions(self) -> dict: return { f.name(): { - 'type': 'Conv', - 'parameters': {'type': 'Wav', 'filename': f.filename, 'channel': f.channel_in_file} + "type": "Conv", + "parameters": { + "type": "Wav", + "filename": f.filename, + "channel": f.channel_in_file, + }, } for f in self._filters } @@ -102,13 +129,14 @@ def _input_delay_pipeline_steps(self) -> list[dict]: def _delay_pipeline_steps(self, delays: list[int]) -> list[dict]: return [ { - 'type': 'Filter', - 'channel': channel, - 'names': [self._delay_name(delay)], - 'bypassed': None, - 'description': None, + "type": "Filter", + "channel": channel, + "names": [self._delay_name(delay)], + "bypassed": None, + "description": None, } - for channel, delay in enumerate(delays) if delay != 0 + for channel, delay in enumerate(delays) + if delay != 0 ] def _output_delay_pipeline_steps(self) -> list[dict]: @@ -116,52 +144,72 @@ def _output_delay_pipeline_steps(self) -> list[dict]: def _mixer_in(self) -> dict: return { - 'Mixer in': { - 'channels': { - 'in': self._input_channels, - 'out': max(1, len(self._filters)) + "Mixer in": { + "channels": { + "in": self._input_channels, + "out": max(1, len(self._filters)), }, - 'mapping': [ - {'dest': f.channel, - 'sources': [ - {'channel': channel, 'gain': factor, 'scale': 'linear', 'inverted': invert} - for (channel, factor, invert) in f.input_channels - ]} + "mapping": [ + { + "dest": f.channel, + "sources": [ + { + "channel": channel, + "gain": factor, + "scale": "linear", + "inverted": invert, + } + for (channel, factor, invert) in f.input_channels + ], + } for f in self._filters - ] + ], } } def _mixer_out(self) -> dict: return { - 'Mixer out': { - 'channels': { - 'in': max(1, len(self._filters)), - 'out': self._output_channels + "Mixer out": { + "channels": { + "in": max(1, len(self._filters)), + "out": self._output_channels, }, - 'mapping': [ - {'dest': output_channel, - 'sources': [ - {'channel': f.channel, 'gain': factor, 'scale': 'linear', 'inverted': invert} - for f in self._filters - for (channel, factor, invert) in f.output_channels - if channel == output_channel - ]} + "mapping": [ + { + "dest": output_channel, + "sources": [ + { + "channel": f.channel, + "gain": factor, + "scale": "linear", + "inverted": invert, + } + for f in self._filters + for (channel, factor, invert) in f.output_channels + if channel == output_channel + ], + } for output_channel in range(self._output_channels) - ] + ], } } @staticmethod def _mixer_in_pipeline_step() -> list[dict]: - return [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}] + return [{"type": "Mixer", "name": "Mixer in", "description": None}] @staticmethod def _mixer_out_pipeline_step() -> list[dict]: - return [{'type': 'Mixer', 'name': 'Mixer out', 'description': None}] + return [{"type": "Mixer", "name": "Mixer out", "description": None}] def _filter_pipeline_steps(self) -> list[dict]: return [ - {'type': 'Filter', 'channel': f.channel, 'names': [f.name()], 'bypassed': None, 'description': None} + { + "type": "Filter", + "channel": f.channel, + "names": [f.name()], + "bypassed": None, + "description": None, + } for f in self._filters ] diff --git a/backend/convolver_config_import_test.py b/backend/convolver_config_import_test.py index 22347c1..edf2296 100644 --- a/backend/convolver_config_import_test.py +++ b/backend/convolver_config_import_test.py @@ -1,6 +1,10 @@ from unittest import TestCase from textwrap import dedent -from convolver_config_import import ConvolverConfig, filename_of_path, channels_factors_and_inversions_as_list +from convolver_config_import import ( + ConvolverConfig, + filename_of_path, + channels_factors_and_inversions_as_list, +) def clean_multi_line_string(multiline_text: str): @@ -8,61 +12,88 @@ def clean_multi_line_string(multiline_text: str): :param multiline_text: :return: the text without the first blank line and indentation """ - return dedent(multiline_text.removeprefix('\n')) + return dedent(multiline_text.removeprefix("\n")) class Test(TestCase): - def test_filename_of_path(self): - self.assertEqual('File.wav', filename_of_path('File.wav')) - self.assertEqual('File.wav', filename_of_path('/some/path/File.wav')) - self.assertEqual('File.wav', filename_of_path('C:\\some\\path\\File.wav')) + self.assertEqual("File.wav", filename_of_path("File.wav")) + self.assertEqual("File.wav", filename_of_path("/some/path/File.wav")) + self.assertEqual("File.wav", filename_of_path("C:\\some\\path\\File.wav")) def test_channels_factors_and_inversions_as_list(self): self.assertEqual( - channels_factors_and_inversions_as_list("0.0 1.1 -9.9 -0.0"), - [(0, 0.0, False), (1, 0.1, False), (9, 0.9, True), (0, 0, True)] + channels_factors_and_inversions_as_list("0.0 1.1 -9.9"), + [(0, 1.0, False), (1, 0.1, False), (9, 0.9, True)], + ) + # Straigth inversion + # Note, the Convolver documentation says to use + # -0.99999 and not -0.0 for this. + self.assertEqual( + channels_factors_and_inversions_as_list("-0.0 -0.99999"), + [(0, 1.0, True), (0, 0.99999, True)], ) def test_samplerate_is_imported(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 96000 1 2 0 0 0 - """) + """ + ) conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual(conf['devices'], {'samplerate': 96000}) + self.assertEqual(conf["devices"], {"samplerate": 96000}) def test_delays_and_mixers_are_imported(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 96000 2 3 0 3 0 4 - """) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual( - conf['filters'], - {'Delay3': {'type': 'Delay', 'parameters': {'delay': 3, 'unit': 'ms', 'subsample': False}}, - 'Delay4': {'type': 'Delay', 'parameters': {'delay': 4, 'unit': 'ms', 'subsample': False}}} - ) - self.assertEqual( - conf['mixers']['Mixer in']['channels'], - {'in': 2, 'out': 1} + """ ) + conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - conf['mixers']['Mixer out']['channels'], - {'in': 1, 'out': 3} + conf["filters"], + { + "Delay3": { + "type": "Delay", + "parameters": {"delay": 3, "unit": "ms", "subsample": False}, + }, + "Delay4": { + "type": "Delay", + "parameters": {"delay": 4, "unit": "ms", "subsample": False}, + }, + }, ) + self.assertEqual(conf["mixers"]["Mixer in"]["channels"], {"in": 2, "out": 1}) + self.assertEqual(conf["mixers"]["Mixer out"]["channels"], {"in": 1, "out": 3}) self.assertEqual( - conf['pipeline'], - [{'type': 'Filter', 'channel': 0, 'names': ['Delay3'], 'bypassed': None, 'description': None}, - {'type': 'Mixer', 'name': 'Mixer in', 'description': None}, - {'type': 'Mixer', 'name': 'Mixer out', 'description': None}, - {'type': 'Filter', 'channel': 1, 'names': ['Delay4'], 'bypassed': None, 'description': None}] + conf["pipeline"], + [ + { + "type": "Filter", + "channel": 0, + "names": ["Delay3"], + "bypassed": None, + "description": None, + }, + {"type": "Mixer", "name": "Mixer in", "description": None}, + {"type": "Mixer", "name": "Mixer out", "description": None}, + { + "type": "Filter", + "channel": 1, + "names": ["Delay4"], + "bypassed": None, + "description": None, + }, + ], ) def test_simple_impulse_response(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 0 1 1 0 0 0 @@ -70,23 +101,36 @@ def test_simple_impulse_response(self): 0 0.0 0.0 - """) + """ + ) conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - conf['filters'], - {'IR.wav-0': { - 'type': 'Conv', - 'parameters': {'type': 'Wav', 'filename': 'IR.wav', 'channel': 0}}} + conf["filters"], + { + "IR.wav-0": { + "type": "Conv", + "parameters": {"type": "Wav", "filename": "IR.wav", "channel": 0}, + } + }, ) self.assertEqual( - conf['pipeline'], - [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}, - {'type': 'Filter', 'channel': 0, 'names': ['IR.wav-0'], 'bypassed': None, 'description': None}, - {'type': 'Mixer', 'name': 'Mixer out', 'description': None}] + conf["pipeline"], + [ + {"type": "Mixer", "name": "Mixer in", "description": None}, + { + "type": "Filter", + "channel": 0, + "names": ["IR.wav-0"], + "bypassed": None, + "description": None, + }, + {"type": "Mixer", "name": "Mixer out", "description": None}, + ], ) def test_path_is_ignored_for_impulse_response_files(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 0 1 1 0 0 0 @@ -102,14 +146,22 @@ def test_path_is_ignored_for_impulse_response_files(self): 0 0.0 0.0 - """) + """ + ) conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual(conf['filters']['IR1.wav-0']['parameters']['filename'], 'IR1.wav') - self.assertEqual(conf['filters']['IR2.wav-0']['parameters']['filename'], 'IR2.wav') - self.assertEqual(conf['filters']['IR3.wav-0']['parameters']['filename'], 'IR3.wav') + self.assertEqual( + conf["filters"]["IR1.wav-0"]["parameters"]["filename"], "IR1.wav" + ) + self.assertEqual( + conf["filters"]["IR2.wav-0"]["parameters"]["filename"], "IR2.wav" + ) + self.assertEqual( + conf["filters"]["IR3.wav-0"]["parameters"]["filename"], "IR3.wav" + ) def test_wav_file_with_multiple_impulse_responses(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 0 1 1 0 0 0 @@ -121,13 +173,15 @@ def test_wav_file_with_multiple_impulse_responses(self): 1 0.0 0.0 - """) + """ + ) conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual(conf['filters']['IR.wav-0']['parameters']['channel'], 0) - self.assertEqual(conf['filters']['IR.wav-1']['parameters']['channel'], 1) + self.assertEqual(conf["filters"]["IR.wav-0"]["parameters"]["channel"], 0) + self.assertEqual(conf["filters"]["IR.wav-1"]["parameters"]["channel"], 1) def test_impulse_responses_are_mapped_to_correct_channels(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 0 1 1 0 0 0 @@ -139,18 +193,34 @@ def test_impulse_responses_are_mapped_to_correct_channels(self): 0 0.0 0.0 - """) + """ + ) conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - conf['pipeline'], - [{'type': 'Mixer', 'name': 'Mixer in', 'description': None}, - {'type': 'Filter', 'channel': 0, 'names': ['IR1.wav-0'], 'bypassed': None, 'description': None}, - {'type': 'Filter', 'channel': 1, 'names': ['IR2.wav-0'], 'bypassed': None, 'description': None}, - {'type': 'Mixer', 'name': 'Mixer out', 'description': None}] + conf["pipeline"], + [ + {"type": "Mixer", "name": "Mixer in", "description": None}, + { + "type": "Filter", + "channel": 0, + "names": ["IR1.wav-0"], + "bypassed": None, + "description": None, + }, + { + "type": "Filter", + "channel": 1, + "names": ["IR2.wav-0"], + "bypassed": None, + "description": None, + }, + {"type": "Mixer", "name": "Mixer out", "description": None}, + ], ) def test_impulse_response_with_input_scaling(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 0 2 2 0 0 0 0 0 @@ -166,30 +236,72 @@ def test_impulse_response_with_input_scaling(self): 2 -1.5 -0.4 0.0 - """) + """ + ) conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - conf['mixers']['Mixer in'], - {'channels': {'in': 2, 'out': 3}, - 'mapping': [ - {'dest': 0, - 'sources': [ - {'channel': 0, 'gain': 0.0, 'scale': 'linear', 'inverted': False}, - {'channel': 1, 'gain': 0.1, 'scale': 'linear', 'inverted': False}]}, - {'dest': 1, - 'sources': [ - {'channel': 0, 'gain': 0.2, 'scale': 'linear', 'inverted': False}, - {'channel': 1, 'gain': 0.3, 'scale': 'linear', 'inverted': False}]}, - {'dest': 2, - 'sources': [ - {'channel': 1, 'gain': 0.5, 'scale': 'linear', 'inverted': True}, - {'channel': 0, 'gain': 0.4, 'scale': 'linear', 'inverted': True} - ]}, - ]} + conf["mixers"]["Mixer in"], + { + "channels": {"in": 2, "out": 3}, + "mapping": [ + { + "dest": 0, + "sources": [ + { + "channel": 0, + "gain": 1.0, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.1, + "scale": "linear", + "inverted": False, + }, + ], + }, + { + "dest": 1, + "sources": [ + { + "channel": 0, + "gain": 0.2, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.3, + "scale": "linear", + "inverted": False, + }, + ], + }, + { + "dest": 2, + "sources": [ + { + "channel": 1, + "gain": 0.5, + "scale": "linear", + "inverted": True, + }, + { + "channel": 0, + "gain": 0.4, + "scale": "linear", + "inverted": True, + }, + ], + }, + ], + }, ) def test_impulse_response_with_output_scaling(self): - convolver_config = clean_multi_line_string(""" + convolver_config = clean_multi_line_string( + """ 0 2 2 0 0 0 0 0 @@ -205,21 +317,60 @@ def test_impulse_response_with_output_scaling(self): 2 0.0 -1.5 -0.4 - """) + """ + ) conf = ConvolverConfig(convolver_config).to_object() self.assertEqual( - conf['mixers']['Mixer out'], - {'channels': {'in': 3, 'out': 2}, - 'mapping': [ - {'dest': 0, - 'sources': [ - {'channel': 0, 'gain': 0.0, 'scale': 'linear', 'inverted': False}, - {'channel': 1, 'gain': 0.2, 'scale': 'linear', 'inverted': False}, - {'channel': 2, 'gain': 0.4, 'scale': 'linear', 'inverted': True}]}, - {'dest': 1, - 'sources': [ - {'channel': 0, 'gain': 0.1, 'scale': 'linear', 'inverted': False}, - {'channel': 1, 'gain': 0.3, 'scale': 'linear', 'inverted': False}, - {'channel': 2, 'gain': 0.5, 'scale': 'linear', 'inverted': True}]}, - ]} + conf["mixers"]["Mixer out"], + { + "channels": {"in": 3, "out": 2}, + "mapping": [ + { + "dest": 0, + "sources": [ + { + "channel": 0, + "gain": 1.0, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.2, + "scale": "linear", + "inverted": False, + }, + { + "channel": 2, + "gain": 0.4, + "scale": "linear", + "inverted": True, + }, + ], + }, + { + "dest": 1, + "sources": [ + { + "channel": 0, + "gain": 0.1, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.3, + "scale": "linear", + "inverted": False, + }, + { + "channel": 2, + "gain": 0.5, + "scale": "linear", + "inverted": True, + }, + ], + }, + ], + }, ) From c969527196402069591ebda00bc0b8105a5c5338 Mon Sep 17 00:00:00 2001 From: Henrik Date: Sat, 20 Jan 2024 21:46:37 +0100 Subject: [PATCH 07/37] Run tests from repo root --- .github/workflows/build.yml | 2 +- backend/convolver_config_import_test.py | 2 +- backend/filters_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47f0143..5805bce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: with: python-version: '3.12' - name: Run python tests - run: cd backend ; python3 -m unittest discover -p "*_test.py" + run: python3 -m unittest discover -p "*_test.py" - name: Install template render dependencies run: | diff --git a/backend/convolver_config_import_test.py b/backend/convolver_config_import_test.py index edf2296..f82af67 100644 --- a/backend/convolver_config_import_test.py +++ b/backend/convolver_config_import_test.py @@ -1,6 +1,6 @@ from unittest import TestCase from textwrap import dedent -from convolver_config_import import ( +from backend.convolver_config_import import ( ConvolverConfig, filename_of_path, channels_factors_and_inversions_as_list, diff --git a/backend/filters_test.py b/backend/filters_test.py index c5486bb..8bb534b 100644 --- a/backend/filters_test.py +++ b/backend/filters_test.py @@ -1,5 +1,5 @@ from unittest import TestCase -from filters import filter_plot_options, pipeline_step_plot_options +from backend.filters import filter_plot_options, pipeline_step_plot_options class FiltersTest(TestCase): From 74bad94843c10303e863a21026519b7295b32c09 Mon Sep 17 00:00:00 2001 From: Henrik Date: Sat, 20 Jan 2024 23:52:01 +0100 Subject: [PATCH 08/37] Reduce duplication --- backend/convolver_config_import.py | 72 ++++++++++++++---------------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/backend/convolver_config_import.py b/backend/convolver_config_import.py index 788d560..4f53d18 100644 --- a/backend/convolver_config_import.py +++ b/backend/convolver_config_import.py @@ -38,6 +38,31 @@ def channels_factors_and_inversions_as_list( ] +def make_filter_step(channel: int, names: list[str]) -> dict: + return { + "type": "Filter", + "channel": channel, + "names": names, + "bypassed": None, + "description": None, + } + + +def make_mixer_mapping(input_channels: list[tuple], output_channel: int) -> dict: + return { + "dest": output_channel, + "sources": [ + { + "channel": channel, + "gain": factor, + "scale": "linear", + "inverted": invert, + } + for (channel, factor, invert) in input_channels + ], + } + + class Filter: filename: str channel: int @@ -128,13 +153,7 @@ def _input_delay_pipeline_steps(self) -> list[dict]: def _delay_pipeline_steps(self, delays: list[int]) -> list[dict]: return [ - { - "type": "Filter", - "channel": channel, - "names": [self._delay_name(delay)], - "bypassed": None, - "description": None, - } + make_filter_step(channel, [self._delay_name(delay)]) for channel, delay in enumerate(delays) if delay != 0 ] @@ -150,18 +169,7 @@ def _mixer_in(self) -> dict: "out": max(1, len(self._filters)), }, "mapping": [ - { - "dest": f.channel, - "sources": [ - { - "channel": channel, - "gain": factor, - "scale": "linear", - "inverted": invert, - } - for (channel, factor, invert) in f.input_channels - ], - } + make_mixer_mapping(f.input_channels, f.channel) for f in self._filters ], } @@ -175,20 +183,15 @@ def _mixer_out(self) -> dict: "out": self._output_channels, }, "mapping": [ - { - "dest": output_channel, - "sources": [ - { - "channel": f.channel, - "gain": factor, - "scale": "linear", - "inverted": invert, - } + make_mixer_mapping( + [ + (f.channel, factor, invert) for f in self._filters for (channel, factor, invert) in f.output_channels if channel == output_channel ], - } + output_channel, + ) for output_channel in range(self._output_channels) ], } @@ -203,13 +206,4 @@ def _mixer_out_pipeline_step() -> list[dict]: return [{"type": "Mixer", "name": "Mixer out", "description": None}] def _filter_pipeline_steps(self) -> list[dict]: - return [ - { - "type": "Filter", - "channel": f.channel, - "names": [f.name()], - "bypassed": None, - "description": None, - } - for f in self._filters - ] + return [make_filter_step(f.channel, [f.name()]) for f in self._filters] From 3116f3f50bec79e0cad9057e644b5d34d9176590 Mon Sep 17 00:00:00 2001 From: Henrik Date: Sun, 21 Jan 2024 10:36:45 +0100 Subject: [PATCH 09/37] Convert filter tests to pytest --- backend/filters_test.py | 174 ---------------------------------------- tests/test_filters.py | 152 +++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 174 deletions(-) delete mode 100644 backend/filters_test.py create mode 100644 tests/test_filters.py diff --git a/backend/filters_test.py b/backend/filters_test.py deleted file mode 100644 index 8bb534b..0000000 --- a/backend/filters_test.py +++ /dev/null @@ -1,174 +0,0 @@ -from unittest import TestCase -from backend.filters import filter_plot_options, pipeline_step_plot_options - - -class FiltersTest(TestCase): - - def test_filter_plot_options_with_samplerate(self): - self.assertEqual( - filter_plot_options( - ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], - "filter_$samplerate$_2" - ), - [ - {"name": "filter_44100_2", "samplerate": 44100}, - {"name": "filter_48000_2", "samplerate": 48000} - ] - ) - - def test_filter_plot_options_with_channels(self): - self.assertEqual( - filter_plot_options( - ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], - "filter_44100_$channels$" - ), - [ - {"name": "filter_44100_2", "channels": 2}, - {"name": "filter_44100_8", "channels": 8} - ] - ) - - def test_filter_plot_options_with_samplerate_and_channels(self): - self.assertEqual( - filter_plot_options( - ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], - "filter_$samplerate$_$channels$" - ), - [ - {"name": "filter_44100_2", "samplerate": 44100, "channels": 2}, - {"name": "filter_44100_8", "samplerate": 44100, "channels": 8}, - {"name": "filter_48000_2", "samplerate": 48000, "channels": 2}, - {"name": "filter_48000_8", "samplerate": 48000, "channels": 8} - ] - ) - self.assertEqual( - filter_plot_options( - ["filter_2_44100", "filter_8_44100", "filter_2_48000", "filter_8_48000"], - "filter_$channels$_$samplerate$" - ), - [ - {"name": "filter_2_44100", "samplerate": 44100, "channels": 2}, - {"name": "filter_8_44100", "samplerate": 44100, "channels": 8}, - {"name": "filter_2_48000", "samplerate": 48000, "channels": 2}, - {"name": "filter_8_48000", "samplerate": 48000, "channels": 8} - ] - ) - - def test_filter_plot_options_without_samplerate_and_channels(self): - self.assertEqual( - filter_plot_options( - ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], - "filter_44100_2" - ), - [{"name": "filter_44100_2"}] - ) - - def test_filter_plot_options_handles_filenames_with_brackets(self): - self.assertEqual( - filter_plot_options( - [ - "filter_((44100)_(2))", - "filter_((44100)_(8))", - "filter_((48000)_(2))", - "filter_((48000)_(8))" - ], - "filter_(($samplerate$)_($channels$))" - ), - [ - {"name": "filter_((44100)_(2))", "samplerate": 44100, "channels": 2}, - {"name": "filter_((44100)_(8))", "samplerate": 44100, "channels": 8}, - {"name": "filter_((48000)_(2))", "samplerate": 48000, "channels": 2}, - {"name": "filter_((48000)_(8))", "samplerate": 48000, "channels": 8} - ] - ) - - def test_pipeline_step_plot_options_for_only_one_samplerate_and_channel_option(self): - config = { - "devices": { - "samplerate": 44100, - "capture": {"channels": 2} - }, - "filters": { - "Filter1": { - "type": "Conv", - "parameters": { - "type": "Raw", - "filename": "../coeffs/filter-44100-2" - } - }, - "Filter2": { - "type": "Conv", - "parameters": { - "type": "Wav", - "filename": "../coeffs/filter-$samplerate$-$channels$" - } - }, - "irrelevantFilter": { - "type": "something else", - "parameters": {} - } - }, - "pipeline": [ - { - "channel": 0, - "type": "Filter", - "names": ["Filter1", "Filter2", "irrelevantFilter"] - } - ] - } - filter_file_names = [ - "filter-44100-2", - "filter-44100-8", - "filter-48000-2", - "filter-48000-8" - ] - self.assertEqual( - pipeline_step_plot_options(filter_file_names, config, 0), - [{"name": "44100 Hz - 2 Channels", "samplerate": 44100, "channels": 2}] - ) - - def test_pipeline_step_plot_options_for_many_samplerate_and_channel_options(self): - config = { - "devices": { - "samplerate": 44100, - "capture": {"channels": 2} - }, - "filters": { - "Filter1": { - "type": "Conv", - "parameters": { - "type": "Raw", - "filename": "../coeffs/filter-$samplerate$-$channels$" - } - }, - "Filter2": { - "type": "Conv", - "parameters": { - "type": "Raw", - "filename": "../coeffs/filter-$samplerate$-$channels$" - } - } - }, - "pipeline": [ - { - "channel": 0, - "type": "Filter", - "names": ["Filter1", "Filter2"] - } - ] - } - filter_file_names = [ - "filter-44100-2", - "filter-44100-8", - "filter-48000-2", - "filter-48000-8" - ] - self.assertEqual( - pipeline_step_plot_options(filter_file_names, config, 0), - [ - {"name": "44100 Hz - 2 Channels", "samplerate": 44100, "channels": 2}, - {"name": "44100 Hz - 8 Channels", "samplerate": 44100, "channels": 8}, - {"name": "48000 Hz - 2 Channels", "samplerate": 48000, "channels": 2}, - {"name": "48000 Hz - 8 Channels", "samplerate": 48000, "channels": 8} - ] - ) \ No newline at end of file diff --git a/tests/test_filters.py b/tests/test_filters.py new file mode 100644 index 0000000..f5b1bad --- /dev/null +++ b/tests/test_filters.py @@ -0,0 +1,152 @@ +from backend.filters import filter_plot_options, pipeline_step_plot_options + + +def test_filter_plot_options_with_samplerate(): + result = filter_plot_options( + ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], + "filter_$samplerate$_2", + ) + expected = [ + {"name": "filter_44100_2", "samplerate": 44100}, + {"name": "filter_48000_2", "samplerate": 48000}, + ] + assert result == expected + + +def test_filter_plot_options_with_channels(): + result = filter_plot_options( + ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], + "filter_44100_$channels$", + ) + expected = [ + {"name": "filter_44100_2", "channels": 2}, + {"name": "filter_44100_8", "channels": 8}, + ] + assert result == expected + + +def test_filter_plot_options_with_samplerate_and_channels(): + result1 = filter_plot_options( + ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], + "filter_$samplerate$_$channels$", + ) + expected1 = [ + {"name": "filter_44100_2", "samplerate": 44100, "channels": 2}, + {"name": "filter_44100_8", "samplerate": 44100, "channels": 8}, + {"name": "filter_48000_2", "samplerate": 48000, "channels": 2}, + {"name": "filter_48000_8", "samplerate": 48000, "channels": 8}, + ] + assert result1 == expected1 + + result2 = filter_plot_options( + ["filter_2_44100", "filter_8_44100", "filter_2_48000", "filter_8_48000"], + "filter_$channels$_$samplerate$", + ) + expected2 = [ + {"name": "filter_2_44100", "samplerate": 44100, "channels": 2}, + {"name": "filter_8_44100", "samplerate": 44100, "channels": 8}, + {"name": "filter_2_48000", "samplerate": 48000, "channels": 2}, + {"name": "filter_8_48000", "samplerate": 48000, "channels": 8}, + ] + assert result2 == expected2 + + +def test_filter_plot_options_without_samplerate_and_channels(): + result = filter_plot_options( + ["filter_44100_2", "filter_44100_8", "filter_48000_2", "filter_48000_8"], + "filter_44100_2", + ) + expected = [{"name": "filter_44100_2"}] + assert result == expected + + +def test_filter_plot_options_handles_filenames_with_brackets(): + expected = filter_plot_options( + [ + "filter_((44100)_(2))", + "filter_((44100)_(8))", + "filter_((48000)_(2))", + "filter_((48000)_(8))", + ], + "filter_(($samplerate$)_($channels$))", + ) + result = [ + {"name": "filter_((44100)_(2))", "samplerate": 44100, "channels": 2}, + {"name": "filter_((44100)_(8))", "samplerate": 44100, "channels": 8}, + {"name": "filter_((48000)_(2))", "samplerate": 48000, "channels": 2}, + {"name": "filter_((48000)_(8))", "samplerate": 48000, "channels": 8}, + ] + assert result == expected + + +def test_pipeline_step_plot_options_for_only_one_samplerate_and_channel_option(): + config = { + "devices": {"samplerate": 44100, "capture": {"channels": 2}}, + "filters": { + "Filter1": { + "type": "Conv", + "parameters": {"type": "Raw", "filename": "../coeffs/filter-44100-2"}, + }, + "Filter2": { + "type": "Conv", + "parameters": { + "type": "Wav", + "filename": "../coeffs/filter-$samplerate$-$channels$", + }, + }, + "irrelevantFilter": {"type": "something else", "parameters": {}}, + }, + "pipeline": [ + { + "channel": 0, + "type": "Filter", + "names": ["Filter1", "Filter2", "irrelevantFilter"], + } + ], + } + filter_file_names = [ + "filter-44100-2", + "filter-44100-8", + "filter-48000-2", + "filter-48000-8", + ] + result = pipeline_step_plot_options(filter_file_names, config, 0) + expected = [{"name": "44100 Hz - 2 Channels", "samplerate": 44100, "channels": 2}] + assert result == expected + + +def test_pipeline_step_plot_options_for_many_samplerate_and_channel_options(): + config = { + "devices": {"samplerate": 44100, "capture": {"channels": 2}}, + "filters": { + "Filter1": { + "type": "Conv", + "parameters": { + "type": "Raw", + "filename": "../coeffs/filter-$samplerate$-$channels$", + }, + }, + "Filter2": { + "type": "Conv", + "parameters": { + "type": "Raw", + "filename": "../coeffs/filter-$samplerate$-$channels$", + }, + }, + }, + "pipeline": [{"channel": 0, "type": "Filter", "names": ["Filter1", "Filter2"]}], + } + filter_file_names = [ + "filter-44100-2", + "filter-44100-8", + "filter-48000-2", + "filter-48000-8", + ] + result = pipeline_step_plot_options(filter_file_names, config, 0) + expected = [ + {"name": "44100 Hz - 2 Channels", "samplerate": 44100, "channels": 2}, + {"name": "44100 Hz - 8 Channels", "samplerate": 44100, "channels": 8}, + {"name": "48000 Hz - 2 Channels", "samplerate": 48000, "channels": 2}, + {"name": "48000 Hz - 8 Channels", "samplerate": 48000, "channels": 8}, + ] + assert result == expected From c5dc4430374aafffcba56fa8bc3d7934e4c38fe0 Mon Sep 17 00:00:00 2001 From: Henrik Date: Mon, 22 Jan 2024 21:53:47 +0100 Subject: [PATCH 10/37] Convert existing tests to pytest, add simple api tests --- .github/workflows/build.yml | 4 +- README.md | 5 +- backend/convolver_config_import_test.py | 376 ------------------------ main.py | 66 +++-- tests/test_basic_api.py | 66 +++++ tests/test_convolver_config_import.py | 373 +++++++++++++++++++++++ 6 files changed, 483 insertions(+), 407 deletions(-) delete mode 100644 backend/convolver_config_import_test.py create mode 100644 tests/test_basic_api.py create mode 100644 tests/test_convolver_config_import.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5805bce..6480241 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,8 +47,10 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.12' + - name: Set up pytest + run: python3 -m pip install pytest-aiohttp - name: Run python tests - run: python3 -m unittest discover -p "*_test.py" + run: python3 -m pytest - name: Install template render dependencies run: | diff --git a/README.md b/README.md index 5b232df..18fd83d 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,10 @@ When rendering, the versions of the Python dependencies are taken from the file The backend version is read from `backend/version.py`. ### Running the tests +Install the pytest plugin `pytest-aiohttp`. + +Execute the tests with: ```sh -python -m unittest discover -p "*_test.py" +python -m pytest ``` diff --git a/backend/convolver_config_import_test.py b/backend/convolver_config_import_test.py deleted file mode 100644 index f82af67..0000000 --- a/backend/convolver_config_import_test.py +++ /dev/null @@ -1,376 +0,0 @@ -from unittest import TestCase -from textwrap import dedent -from backend.convolver_config_import import ( - ConvolverConfig, - filename_of_path, - channels_factors_and_inversions_as_list, -) - - -def clean_multi_line_string(multiline_text: str): - """ - :param multiline_text: - :return: the text without the first blank line and indentation - """ - return dedent(multiline_text.removeprefix("\n")) - - -class Test(TestCase): - def test_filename_of_path(self): - self.assertEqual("File.wav", filename_of_path("File.wav")) - self.assertEqual("File.wav", filename_of_path("/some/path/File.wav")) - self.assertEqual("File.wav", filename_of_path("C:\\some\\path\\File.wav")) - - def test_channels_factors_and_inversions_as_list(self): - self.assertEqual( - channels_factors_and_inversions_as_list("0.0 1.1 -9.9"), - [(0, 1.0, False), (1, 0.1, False), (9, 0.9, True)], - ) - # Straigth inversion - # Note, the Convolver documentation says to use - # -0.99999 and not -0.0 for this. - self.assertEqual( - channels_factors_and_inversions_as_list("-0.0 -0.99999"), - [(0, 1.0, True), (0, 0.99999, True)], - ) - - def test_samplerate_is_imported(self): - convolver_config = clean_multi_line_string( - """ - 96000 1 2 0 - 0 - 0 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual(conf["devices"], {"samplerate": 96000}) - - def test_delays_and_mixers_are_imported(self): - convolver_config = clean_multi_line_string( - """ - 96000 2 3 0 - 3 - 0 4 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual( - conf["filters"], - { - "Delay3": { - "type": "Delay", - "parameters": {"delay": 3, "unit": "ms", "subsample": False}, - }, - "Delay4": { - "type": "Delay", - "parameters": {"delay": 4, "unit": "ms", "subsample": False}, - }, - }, - ) - self.assertEqual(conf["mixers"]["Mixer in"]["channels"], {"in": 2, "out": 1}) - self.assertEqual(conf["mixers"]["Mixer out"]["channels"], {"in": 1, "out": 3}) - self.assertEqual( - conf["pipeline"], - [ - { - "type": "Filter", - "channel": 0, - "names": ["Delay3"], - "bypassed": None, - "description": None, - }, - {"type": "Mixer", "name": "Mixer in", "description": None}, - {"type": "Mixer", "name": "Mixer out", "description": None}, - { - "type": "Filter", - "channel": 1, - "names": ["Delay4"], - "bypassed": None, - "description": None, - }, - ], - ) - - def test_simple_impulse_response(self): - convolver_config = clean_multi_line_string( - """ - 0 1 1 0 - 0 - 0 - IR.wav - 0 - 0.0 - 0.0 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual( - conf["filters"], - { - "IR.wav-0": { - "type": "Conv", - "parameters": {"type": "Wav", "filename": "IR.wav", "channel": 0}, - } - }, - ) - self.assertEqual( - conf["pipeline"], - [ - {"type": "Mixer", "name": "Mixer in", "description": None}, - { - "type": "Filter", - "channel": 0, - "names": ["IR.wav-0"], - "bypassed": None, - "description": None, - }, - {"type": "Mixer", "name": "Mixer out", "description": None}, - ], - ) - - def test_path_is_ignored_for_impulse_response_files(self): - convolver_config = clean_multi_line_string( - """ - 0 1 1 0 - 0 - 0 - IR1.wav - 0 - 0.0 - 0.0 - C:\\any/path/IR2.wav - 0 - 0.0 - 0.0 - /some/other/path/IR3.wav - 0 - 0.0 - 0.0 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual( - conf["filters"]["IR1.wav-0"]["parameters"]["filename"], "IR1.wav" - ) - self.assertEqual( - conf["filters"]["IR2.wav-0"]["parameters"]["filename"], "IR2.wav" - ) - self.assertEqual( - conf["filters"]["IR3.wav-0"]["parameters"]["filename"], "IR3.wav" - ) - - def test_wav_file_with_multiple_impulse_responses(self): - convolver_config = clean_multi_line_string( - """ - 0 1 1 0 - 0 - 0 - IR.wav - 0 - 0.0 - 0.0 - IR.wav - 1 - 0.0 - 0.0 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual(conf["filters"]["IR.wav-0"]["parameters"]["channel"], 0) - self.assertEqual(conf["filters"]["IR.wav-1"]["parameters"]["channel"], 1) - - def test_impulse_responses_are_mapped_to_correct_channels(self): - convolver_config = clean_multi_line_string( - """ - 0 1 1 0 - 0 - 0 - IR1.wav - 0 - 0.0 - 0.0 - IR2.wav - 0 - 0.0 - 0.0 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual( - conf["pipeline"], - [ - {"type": "Mixer", "name": "Mixer in", "description": None}, - { - "type": "Filter", - "channel": 0, - "names": ["IR1.wav-0"], - "bypassed": None, - "description": None, - }, - { - "type": "Filter", - "channel": 1, - "names": ["IR2.wav-0"], - "bypassed": None, - "description": None, - }, - {"type": "Mixer", "name": "Mixer out", "description": None}, - ], - ) - - def test_impulse_response_with_input_scaling(self): - convolver_config = clean_multi_line_string( - """ - 0 2 2 0 - 0 0 - 0 0 - IR.wav - 0 - 0.0 1.1 - 0.0 - IR.wav - 1 - 0.2 1.3 - 0.0 - IR.wav - 2 - -1.5 -0.4 - 0.0 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual( - conf["mixers"]["Mixer in"], - { - "channels": {"in": 2, "out": 3}, - "mapping": [ - { - "dest": 0, - "sources": [ - { - "channel": 0, - "gain": 1.0, - "scale": "linear", - "inverted": False, - }, - { - "channel": 1, - "gain": 0.1, - "scale": "linear", - "inverted": False, - }, - ], - }, - { - "dest": 1, - "sources": [ - { - "channel": 0, - "gain": 0.2, - "scale": "linear", - "inverted": False, - }, - { - "channel": 1, - "gain": 0.3, - "scale": "linear", - "inverted": False, - }, - ], - }, - { - "dest": 2, - "sources": [ - { - "channel": 1, - "gain": 0.5, - "scale": "linear", - "inverted": True, - }, - { - "channel": 0, - "gain": 0.4, - "scale": "linear", - "inverted": True, - }, - ], - }, - ], - }, - ) - - def test_impulse_response_with_output_scaling(self): - convolver_config = clean_multi_line_string( - """ - 0 2 2 0 - 0 0 - 0 0 - IR.wav - 0 - 0.0 - 0.0 1.1 - IR.wav - 1 - 0.0 - 0.2 1.3 - IR.wav - 2 - 0.0 - -1.5 -0.4 - """ - ) - conf = ConvolverConfig(convolver_config).to_object() - self.assertEqual( - conf["mixers"]["Mixer out"], - { - "channels": {"in": 3, "out": 2}, - "mapping": [ - { - "dest": 0, - "sources": [ - { - "channel": 0, - "gain": 1.0, - "scale": "linear", - "inverted": False, - }, - { - "channel": 1, - "gain": 0.2, - "scale": "linear", - "inverted": False, - }, - { - "channel": 2, - "gain": 0.4, - "scale": "linear", - "inverted": True, - }, - ], - }, - { - "dest": 1, - "sources": [ - { - "channel": 0, - "gain": 0.1, - "scale": "linear", - "inverted": False, - }, - { - "channel": 1, - "gain": 0.3, - "scale": "linear", - "inverted": False, - }, - { - "channel": 2, - "gain": 0.5, - "scale": "linear", - "inverted": True, - }, - ], - }, - ], - }, - ) diff --git a/main.py b/main.py index 3b02e7b..be84e2b 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ from aiohttp import web import logging import sys -from camilladsp import CamillaClient +import camilladsp from camilladsp_plot.validate_config import CamillaValidator from backend.version import VERSION @@ -26,31 +26,39 @@ #logging.warning("warning") #logging.error("error") -app = web.Application(client_max_size=1024 ** 3) # set max upload file size to 1GB -app["config_dir"] = config["config_dir"] -app["coeff_dir"] = config["coeff_dir"] -app["default_config"] = config["default_config"] -app["statefile_path"] = config["statefile_path"] -app["log_file"] = config["log_file"] -app["on_set_active_config"] = config["on_set_active_config"] -app["on_get_active_config"] = config["on_get_active_config"] -app["supported_capture_types"] = config["supported_capture_types"] -app["supported_playback_types"] = config["supported_playback_types"] -app["can_update_active_config"] = config["can_update_active_config"] -setup_routes(app) -setup_static_routes(app) - -app["CAMILLA"] = CamillaClient(config["camilla_host"], config["camilla_port"]) -app["RECONNECT_THREAD"] = None -app["STATUSCACHE"] = { - "backend_version": version_string(VERSION), - "py_cdsp_version": version_string(app["CAMILLA"].versions.library()) - } -app["CACHETIME"] = 0 -camillavalidator = CamillaValidator() -if config["supported_capture_types"] is not None: - camillavalidator.set_supported_capture_types(config["supported_capture_types"]) -if config["supported_playback_types"] is not None: - camillavalidator.set_supported_playback_types(config["supported_playback_types"]) -app["VALIDATOR"] = camillavalidator -web.run_app(app, host=config["bind_address"], port=config["port"]) +def build_app(): + app = web.Application(client_max_size=1024 ** 3) # set max upload file size to 1GB + app["config_dir"] = config["config_dir"] + app["coeff_dir"] = config["coeff_dir"] + app["default_config"] = config["default_config"] + app["statefile_path"] = config["statefile_path"] + app["log_file"] = config["log_file"] + app["on_set_active_config"] = config["on_set_active_config"] + app["on_get_active_config"] = config["on_get_active_config"] + app["supported_capture_types"] = config["supported_capture_types"] + app["supported_playback_types"] = config["supported_playback_types"] + app["can_update_active_config"] = config["can_update_active_config"] + setup_routes(app) + setup_static_routes(app) + + app["CAMILLA"] = camilladsp.CamillaClient(config["camilla_host"], config["camilla_port"]) + app["RECONNECT_THREAD"] = None + app["STATUSCACHE"] = { + "backend_version": version_string(VERSION), + "py_cdsp_version": version_string(app["CAMILLA"].versions.library()) + } + app["CACHETIME"] = 0 + camillavalidator = CamillaValidator() + if config["supported_capture_types"] is not None: + camillavalidator.set_supported_capture_types(config["supported_capture_types"]) + if config["supported_playback_types"] is not None: + camillavalidator.set_supported_playback_types(config["supported_playback_types"]) + app["VALIDATOR"] = camillavalidator + return app + +def main(): + app = build_app() + web.run_app(app, host=config["bind_address"], port=config["port"]) + +if __name__ == "__main__": + main() diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py new file mode 100644 index 0000000..b41b716 --- /dev/null +++ b/tests/test_basic_api.py @@ -0,0 +1,66 @@ +import json +import pytest +from unittest.mock import MagicMock, patch +import asyncio +import pytest +from aiohttp import web + +import main +from backend import views + + +@pytest.fixture +def mock_request(mock_app): + request = MagicMock + request.app = mock_app + yield request + + +@pytest.fixture +def mock_app(): + client = MagicMock() + client_constructor = MagicMock(return_value=client) + client.volume = MagicMock() + client.volume.main = MagicMock(side_effect=[-20.0]) + client.levels = MagicMock + client.levels.capture_peak = MagicMock(side_effect=[[-2.0, -3.0]]) + with patch("camilladsp.CamillaClient", client_constructor): + print(client) + print(client.volume) + print(client.volume.main) + app = main.build_app() + print(app["CAMILLA"]) + yield app + + +@pytest.fixture +def server(event_loop, aiohttp_client, mock_app): + return event_loop.run_until_complete(aiohttp_client(mock_app)) + + +@pytest.mark.asyncio +async def test_read_volume(mock_request): + mock_request.match_info = {"name": "volume"} + reply = await views.get_param(mock_request) + assert reply.body == "-20.0" + + +@pytest.mark.asyncio +async def test_read_peaks(mock_request): + mock_request.match_info = {"name": "capturesignalpeak"} + reply = await views.get_list_param(mock_request) + assert json.loads(reply.body) == [-2.0, -3.0] + + +@pytest.mark.asyncio +async def test_read_volume(server): + resp = await server.get("/api/getparam/volume") + assert resp.status == 200 + assert await resp.text() == "-20.0" + + +@pytest.mark.asyncio +async def test_read_peaks(server): + resp = await server.get("/api/getlistparam/capturesignalpeak") + assert resp.status == 200 + assert await resp.json() == [-2.0, -3.0] diff --git a/tests/test_convolver_config_import.py b/tests/test_convolver_config_import.py new file mode 100644 index 0000000..e2b8a0a --- /dev/null +++ b/tests/test_convolver_config_import.py @@ -0,0 +1,373 @@ +from textwrap import dedent +from backend.convolver_config_import import ( + ConvolverConfig, + filename_of_path, + channels_factors_and_inversions_as_list, +) + + +def clean_multi_line_string(multiline_text: str): + """ + :param multiline_text: + :return: the text without the first blank line and indentation + """ + return dedent(multiline_text.removeprefix("\n")) + + +def test_filename_of_path(): + assert "File.wav" == filename_of_path("File.wav") + assert "File.wav" == filename_of_path("/some/path/File.wav") + assert "File.wav" == filename_of_path("C:\\some\\path\\File.wav") + + +def test_channels_factors_and_inversions_as_list(): + assert channels_factors_and_inversions_as_list("0.0 1.1 -9.9") == [ + (0, 1.0, False), + (1, 0.1, False), + (9, 0.9, True), + ] + # Straight inversion + # Note, the Convolver documentation says to use + # -0.99999 and not -0.0 for this. + assert channels_factors_and_inversions_as_list("-0.0 -0.99999") == [ + (0, 1.0, True), + (0, 0.99999, True), + ] + + +def test_samplerate_is_imported(): + convolver_config = clean_multi_line_string( + """ + 96000 1 2 0 + 0 + 0 + """ + ) + conf = ConvolverConfig(convolver_config).to_object() + assert conf["devices"] == {"samplerate": 96000} + + +def test_delays_and_mixers_are_imported(): + convolver_config = clean_multi_line_string( + """ + 96000 2 3 0 + 3 + 0 4 + """ + ) + expected_filters = { + "Delay3": { + "type": "Delay", + "parameters": {"delay": 3, "unit": "ms", "subsample": False}, + }, + "Delay4": { + "type": "Delay", + "parameters": {"delay": 4, "unit": "ms", "subsample": False}, + }, + } + expected_pipeline = [ + { + "type": "Filter", + "channel": 0, + "names": ["Delay3"], + "bypassed": None, + "description": None, + }, + {"type": "Mixer", "name": "Mixer in", "description": None}, + {"type": "Mixer", "name": "Mixer out", "description": None}, + { + "type": "Filter", + "channel": 1, + "names": ["Delay4"], + "bypassed": None, + "description": None, + }, + ] + + conf = ConvolverConfig(convolver_config).to_object() + + assert conf["filters"] == expected_filters + assert conf["mixers"]["Mixer in"]["channels"] == {"in": 2, "out": 1} + assert conf["mixers"]["Mixer out"]["channels"] == {"in": 1, "out": 3} + assert conf["pipeline"] == expected_pipeline + + +def test_simple_impulse_response(): + convolver_config = clean_multi_line_string( + """ + 0 1 1 0 + 0 + 0 + IR.wav + 0 + 0.0 + 0.0 + """ + ) + + expected_filters = { + "IR.wav-0": { + "type": "Conv", + "parameters": {"type": "Wav", "filename": "IR.wav", "channel": 0}, + } + } + expected_pipeline = [ + {"type": "Mixer", "name": "Mixer in", "description": None}, + { + "type": "Filter", + "channel": 0, + "names": ["IR.wav-0"], + "bypassed": None, + "description": None, + }, + {"type": "Mixer", "name": "Mixer out", "description": None}, + ] + + conf = ConvolverConfig(convolver_config).to_object() + assert conf["pipeline"] == expected_pipeline + assert conf["filters"] == expected_filters + + +def test_path_is_ignored_for_impulse_response_files(): + convolver_config = clean_multi_line_string( + """ + 0 1 1 0 + 0 + 0 + IR1.wav + 0 + 0.0 + 0.0 + C:\\any/path/IR2.wav + 0 + 0.0 + 0.0 + /some/other/path/IR3.wav + 0 + 0.0 + 0.0 + """ + ) + conf = ConvolverConfig(convolver_config).to_object() + assert conf["filters"]["IR1.wav-0"]["parameters"]["filename"] == "IR1.wav" + assert conf["filters"]["IR2.wav-0"]["parameters"]["filename"] == "IR2.wav" + assert conf["filters"]["IR3.wav-0"]["parameters"]["filename"] == "IR3.wav" + + +def test_wav_file_with_multiple_impulse_responses(): + convolver_config = clean_multi_line_string( + """ + 0 1 1 0 + 0 + 0 + IR.wav + 0 + 0.0 + 0.0 + IR.wav + 1 + 0.0 + 0.0 + """ + ) + conf = ConvolverConfig(convolver_config).to_object() + assert conf["filters"]["IR.wav-0"]["parameters"]["channel"] == 0 + assert conf["filters"]["IR.wav-1"]["parameters"]["channel"] == 1 + + +def test_impulse_responses_are_mapped_to_correct_channels(): + convolver_config = clean_multi_line_string( + """ + 0 1 1 0 + 0 + 0 + IR1.wav + 0 + 0.0 + 0.0 + IR2.wav + 0 + 0.0 + 0.0 + """ + ) + + expected = [ + {"type": "Mixer", "name": "Mixer in", "description": None}, + { + "type": "Filter", + "channel": 0, + "names": ["IR1.wav-0"], + "bypassed": None, + "description": None, + }, + { + "type": "Filter", + "channel": 1, + "names": ["IR2.wav-0"], + "bypassed": None, + "description": None, + }, + {"type": "Mixer", "name": "Mixer out", "description": None}, + ] + + conf = ConvolverConfig(convolver_config).to_object() + result = conf["pipeline"] + assert result == expected + + +def test_impulse_response_with_input_scaling(): + convolver_config = clean_multi_line_string( + """ + 0 2 2 0 + 0 0 + 0 0 + IR.wav + 0 + 0.0 1.1 + 0.0 + IR.wav + 1 + 0.2 1.3 + 0.0 + IR.wav + 2 + -1.5 -0.4 + 0.0 + """ + ) + expected = { + "channels": {"in": 2, "out": 3}, + "mapping": [ + { + "dest": 0, + "sources": [ + { + "channel": 0, + "gain": 1.0, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.1, + "scale": "linear", + "inverted": False, + }, + ], + }, + { + "dest": 1, + "sources": [ + { + "channel": 0, + "gain": 0.2, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.3, + "scale": "linear", + "inverted": False, + }, + ], + }, + { + "dest": 2, + "sources": [ + { + "channel": 1, + "gain": 0.5, + "scale": "linear", + "inverted": True, + }, + { + "channel": 0, + "gain": 0.4, + "scale": "linear", + "inverted": True, + }, + ], + }, + ], + } + conf = ConvolverConfig(convolver_config).to_object() + result = conf["mixers"]["Mixer in"] + assert result == expected + + +def test_impulse_response_with_output_scaling(): + convolver_config = clean_multi_line_string( + """ + 0 2 2 0 + 0 0 + 0 0 + IR.wav + 0 + 0.0 + 0.0 1.1 + IR.wav + 1 + 0.0 + 0.2 1.3 + IR.wav + 2 + 0.0 + -1.5 -0.4 + """ + ) + expected_mixer = { + "channels": {"in": 3, "out": 2}, + "mapping": [ + { + "dest": 0, + "sources": [ + { + "channel": 0, + "gain": 1.0, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.2, + "scale": "linear", + "inverted": False, + }, + { + "channel": 2, + "gain": 0.4, + "scale": "linear", + "inverted": True, + }, + ], + }, + { + "dest": 1, + "sources": [ + { + "channel": 0, + "gain": 0.1, + "scale": "linear", + "inverted": False, + }, + { + "channel": 1, + "gain": 0.3, + "scale": "linear", + "inverted": False, + }, + { + "channel": 2, + "gain": 0.5, + "scale": "linear", + "inverted": True, + }, + ], + }, + ], + } + + conf = ConvolverConfig(convolver_config).to_object() + assert conf["mixers"]["Mixer out"] == expected_mixer From f8ebd12bf3a1fa0f6ed3f2b2d40a9afc73a1c36f Mon Sep 17 00:00:00 2001 From: Henrik Date: Mon, 22 Jan 2024 22:02:29 +0100 Subject: [PATCH 11/37] Install deps --- .github/workflows/build.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6480241..788628b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,10 +47,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.12' - - name: Set up pytest - run: python3 -m pip install pytest-aiohttp - - name: Run python tests - run: python3 -m pytest - name: Install template render dependencies run: | @@ -63,6 +59,15 @@ jobs: - name: Clean up run: rm -rf release_automation + - name: Install requirements + run: python3 -m pip install -r requirements.txt + + - name: Set up pytest + run: python3 -m pip install pytest-aiohttp + + - name: Run python tests + run: python3 -m pytest + - name: Download frontend uses: actions/download-artifact@v3 From 94898a5ab5e06bca0aafd37a28a277b1f107fcd8 Mon Sep 17 00:00:00 2001 From: Henrik Date: Mon, 22 Jan 2024 22:30:41 +0100 Subject: [PATCH 12/37] Separate config for test runs --- main.py | 34 +++++++++++++++++----------------- tests/test_basic_api.py | 18 +++++++++++++++++- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/main.py b/main.py index be84e2b..670c4b3 100644 --- a/main.py +++ b/main.py @@ -26,22 +26,22 @@ #logging.warning("warning") #logging.error("error") -def build_app(): +def build_app(backend_config): app = web.Application(client_max_size=1024 ** 3) # set max upload file size to 1GB - app["config_dir"] = config["config_dir"] - app["coeff_dir"] = config["coeff_dir"] - app["default_config"] = config["default_config"] - app["statefile_path"] = config["statefile_path"] - app["log_file"] = config["log_file"] - app["on_set_active_config"] = config["on_set_active_config"] - app["on_get_active_config"] = config["on_get_active_config"] - app["supported_capture_types"] = config["supported_capture_types"] - app["supported_playback_types"] = config["supported_playback_types"] - app["can_update_active_config"] = config["can_update_active_config"] + app["config_dir"] = backend_config["config_dir"] + app["coeff_dir"] = backend_config["coeff_dir"] + app["default_config"] = backend_config["default_config"] + app["statefile_path"] = backend_config["statefile_path"] + app["log_file"] = backend_config["log_file"] + app["on_set_active_config"] = backend_config["on_set_active_config"] + app["on_get_active_config"] = backend_config["on_get_active_config"] + app["supported_capture_types"] = backend_config["supported_capture_types"] + app["supported_playback_types"] = backend_config["supported_playback_types"] + app["can_update_active_config"] = backend_config["can_update_active_config"] setup_routes(app) setup_static_routes(app) - app["CAMILLA"] = camilladsp.CamillaClient(config["camilla_host"], config["camilla_port"]) + app["CAMILLA"] = camilladsp.CamillaClient(backend_config["camilla_host"], backend_config["camilla_port"]) app["RECONNECT_THREAD"] = None app["STATUSCACHE"] = { "backend_version": version_string(VERSION), @@ -49,15 +49,15 @@ def build_app(): } app["CACHETIME"] = 0 camillavalidator = CamillaValidator() - if config["supported_capture_types"] is not None: - camillavalidator.set_supported_capture_types(config["supported_capture_types"]) - if config["supported_playback_types"] is not None: - camillavalidator.set_supported_playback_types(config["supported_playback_types"]) + if backend_config["supported_capture_types"] is not None: + camillavalidator.set_supported_capture_types(backend_config["supported_capture_types"]) + if backend_config["supported_playback_types"] is not None: + camillavalidator.set_supported_playback_types(backend_config["supported_playback_types"]) app["VALIDATOR"] = camillavalidator return app def main(): - app = build_app() + app = build_app(config) web.run_app(app, host=config["bind_address"], port=config["port"]) if __name__ == "__main__": diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index b41b716..d1b09f0 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -8,6 +8,22 @@ import main from backend import views +server_config = { + "camilla_host": "127.0.0.1", + "camilla_port": 1234, + "bind_address": "0.0.0.0", + "port": 5005, + "config_dir": ".", + "coeff_dir": ".", + "default_config": "./default_config.yml", + "statefile_path": "./statefile.yml", + "log_file": None, + "on_set_active_config": None, + "on_get_active_config": None, + "supported_capture_types": None, + "supported_playback_types": None, + "can_update_active_config": False, +} @pytest.fixture def mock_request(mock_app): @@ -28,7 +44,7 @@ def mock_app(): print(client) print(client.volume) print(client.volume.main) - app = main.build_app() + app = main.build_app(server_config) print(app["CAMILLA"]) yield app From 9834aecac93b66338b2007e21f31fb8848e056f7 Mon Sep 17 00:00:00 2001 From: Henrik Date: Mon, 22 Jan 2024 22:54:16 +0100 Subject: [PATCH 13/37] Test status endpoint --- tests/test_basic_api.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index d1b09f0..577333c 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -7,6 +7,7 @@ import main from backend import views +import camilladsp server_config = { "camilla_host": "127.0.0.1", @@ -25,6 +26,7 @@ "can_update_active_config": False, } + @pytest.fixture def mock_request(mock_app): request = MagicMock @@ -40,12 +42,30 @@ def mock_app(): client.volume.main = MagicMock(side_effect=[-20.0]) client.levels = MagicMock client.levels.capture_peak = MagicMock(side_effect=[[-2.0, -3.0]]) + client.levels.levels = MagicMock( + side_effect=[ + { + "capture_rms": [-5.0, -6.0], + "capture_peak": [-2.0, -3.0], + "playback_rms": [-7.0, -8.0], + "playback_peak": [-3.0, -4.0], + } + ] + ) + client.rate = MagicMock() + client.rate.capture = MagicMock(side_effect=[44100]) + client.general = MagicMock() + client.general.state = MagicMock( + side_effect=[camilladsp.camilladsp.ProcessingState.RUNNING] + ) + client.status = MagicMock() + client.status.rate_adjust = MagicMock(side_effect=[1.01]) + client.status.buffer_level = MagicMock(side_effect=[1234]) + client.status.clipped_samples = MagicMock(side_effect=[12]) + client.status.processing_load = MagicMock(side_effect=[0.5]) with patch("camilladsp.CamillaClient", client_constructor): - print(client) - print(client.volume) - print(client.volume.main) app = main.build_app(server_config) - print(app["CAMILLA"]) + app["STATUSCACHE"]["py_cdsp_version"] = "1.2.3" yield app @@ -80,3 +100,11 @@ async def test_read_peaks(server): resp = await server.get("/api/getlistparam/capturesignalpeak") assert resp.status == 200 assert await resp.json() == [-2.0, -3.0] + + +@pytest.mark.asyncio +async def test_read_status(server): + resp = await server.get("/api/status") + assert resp.status == 200 + response = await resp.json() + assert response["cdsp_status"] == "RUNNING" From d45b3c2fedb84dca118ef6e0f4fa2bda19a192b6 Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 23 Jan 2024 21:30:36 +0100 Subject: [PATCH 14/37] Clean up before zipping --- .github/workflows/build.yml | 11 ++++--- backend/views.py | 8 ++--- main.py | 6 ++-- tests/test_basic_api.py | 60 +++++++++++++++++++++++++++++++++---- tests/testfiles/config.yml | 12 ++++++++ tests/testfiles/log.txt | 2 ++ 6 files changed, 83 insertions(+), 16 deletions(-) create mode 100644 tests/testfiles/config.yml create mode 100644 tests/testfiles/log.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 788628b..b97b404 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: name: build path: build - build_be: + build_and_test_be: runs-on: ubuntu-latest needs: build_fe steps: @@ -56,9 +56,6 @@ jobs: - name: Render scripts from templates run: python -m release_automation.render_env_files - - name: Clean up - run: rm -rf release_automation - - name: Install requirements run: python3 -m pip install -r requirements.txt @@ -68,6 +65,12 @@ jobs: - name: Run python tests run: python3 -m pytest + - name: Clean up + run: | + rm -rf backend/__pycache__ + rm -rf release_automation + rm -rf tests + - name: Download frontend uses: actions/download-artifact@v3 diff --git a/backend/views.py b/backend/views.py index 621da5c..c68e06e 100644 --- a/backend/views.py +++ b/backend/views.py @@ -56,9 +56,9 @@ async def get_status(request): to the camilladsp process. """ cdsp = request.app["CAMILLA"] - reconnect_thread = request.app["RECONNECT_THREAD"] + reconnect_thread = request.app["STORE"]["reconnect_thread"] cache = request.app["STATUSCACHE"] - cachetime = request.app["CACHETIME"] + cachetime = request.app["STORE"]["cache_time"] try: levels_since = float(request.query.get("since")) except: @@ -81,7 +81,7 @@ async def get_status(request): now = time.time() # These values don't change that fast, let's update them only once per second. if now - cachetime > 1.0: - request.app["CACHETIME"] = now + request.app["STORE"]["cache_time"] = now cache.update({ "capturerate": cdsp.rate.capture(), "rateadjust": cdsp.status.rate_adjust(), @@ -97,7 +97,7 @@ async def get_status(request): cache.update(OFFLINE_CACHE) reconnect_thread = threading.Thread(target=_reconnect, args=(cdsp, cache), daemon=True) reconnect_thread.start() - request.app["RECONNECT_THREAD"] = reconnect_thread + request.app["STORE"]["reconnect_thread"] = reconnect_thread return web.json_response(cache) diff --git a/main.py b/main.py index 670c4b3..7653cc1 100644 --- a/main.py +++ b/main.py @@ -42,12 +42,14 @@ def build_app(backend_config): setup_static_routes(app) app["CAMILLA"] = camilladsp.CamillaClient(backend_config["camilla_host"], backend_config["camilla_port"]) - app["RECONNECT_THREAD"] = None app["STATUSCACHE"] = { "backend_version": version_string(VERSION), "py_cdsp_version": version_string(app["CAMILLA"].versions.library()) } - app["CACHETIME"] = 0 + app["STORE"] = {} + app["STORE"]["reconnect_thread"] = None + app["STORE"]["cache_time"] = 0 + camillavalidator = CamillaValidator() if backend_config["supported_capture_types"] is not None: camillavalidator.set_supported_capture_types(backend_config["supported_capture_types"]) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 577333c..77cd1df 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -4,26 +4,31 @@ import asyncio import pytest from aiohttp import web +import os +import yaml import main from backend import views import camilladsp +TESTFILE_DIR = os.path.join(os.path.dirname(__file__), "testfiles") +SAMPLE_CONFIG = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "config.yml"))) + server_config = { "camilla_host": "127.0.0.1", "camilla_port": 1234, "bind_address": "0.0.0.0", "port": 5005, - "config_dir": ".", - "coeff_dir": ".", - "default_config": "./default_config.yml", - "statefile_path": "./statefile.yml", - "log_file": None, + "config_dir": TESTFILE_DIR, + "coeff_dir": TESTFILE_DIR, + "default_config": os.path.join(TESTFILE_DIR, "config.yml"), + "statefile_path": os.path.join(TESTFILE_DIR, "statefile.yml"), + "log_file": os.path.join(TESTFILE_DIR, "log.txt"), "on_set_active_config": None, "on_get_active_config": None, "supported_capture_types": None, "supported_playback_types": None, - "can_update_active_config": False, + "can_update_active_config": True, } @@ -40,8 +45,11 @@ def mock_app(): client_constructor = MagicMock(return_value=client) client.volume = MagicMock() client.volume.main = MagicMock(side_effect=[-20.0]) + client.mute = MagicMock() + client.mute.main = MagicMock(side_effect=[False]) client.levels = MagicMock client.levels.capture_peak = MagicMock(side_effect=[[-2.0, -3.0]]) + client.levels.playback_peak = MagicMock(side_effect=[[-2.5, -3.5]]) client.levels.levels = MagicMock( side_effect=[ { @@ -58,11 +66,22 @@ def mock_app(): client.general.state = MagicMock( side_effect=[camilladsp.camilladsp.ProcessingState.RUNNING] ) + client.general.list_capture_devices = MagicMock(side_effect=[[ + ["hw:Aaaa,0,0", "Dev A"], + ["hw:Bbbb,0,0", "Dev B"] + ]]) + client.general.list_playback_devices = MagicMock(side_effect=[[ + ["hw:Cccc,0,0", "Dev C"], + ["hw:Dddd,0,0", "Dev D"] + ]]) + client.general.supported_device_types = MagicMock(side_effect = [["Alsa", "Wasapi"]]) client.status = MagicMock() client.status.rate_adjust = MagicMock(side_effect=[1.01]) client.status.buffer_level = MagicMock(side_effect=[1234]) client.status.clipped_samples = MagicMock(side_effect=[12]) client.status.processing_load = MagicMock(side_effect=[0.5]) + client.config = MagicMock() + client.config.active = MagicMock(side_effect=[SAMPLE_CONFIG]) with patch("camilladsp.CamillaClient", client_constructor): app = main.build_app(server_config) app["STATUSCACHE"]["py_cdsp_version"] = "1.2.3" @@ -108,3 +127,32 @@ async def test_read_status(server): assert resp.status == 200 response = await resp.json() assert response["cdsp_status"] == "RUNNING" + + +@pytest.mark.parametrize( + "endpoint, parameters", + [ + ("/api/status", None), + ("/api/getparam/mute", None), + ("/api/getlistparam/playbacksignalpeak", None), + ("/api/getconfig", None), + ("/api/getactiveconfigfile", None), + ("/api/getdefaultconfigfile", None), + ("/api/storedconfigs", None), + ("/api/storedcoeffs", None), + ("/api/defaultsforcoeffs", {"file": "test.wav"}), + ("/api/guiconfig", None), + ("/api/getconfigfile", {"name": "config.yml"}), + ("/api/logfile", None), + ("/api/capturedevices/alsa", None), + ("/api/playbackdevices/alsa", None), + ("/api/backends", None), + ], +) +@pytest.mark.asyncio +async def test_all_get_endpoints_ok(server, endpoint, parameters): + if parameters: + resp = await server.get(endpoint, params=parameters) + else: + resp = await server.get(endpoint) + assert resp.status == 200 diff --git a/tests/testfiles/config.yml b/tests/testfiles/config.yml new file mode 100644 index 0000000..cb5d0e7 --- /dev/null +++ b/tests/testfiles/config.yml @@ -0,0 +1,12 @@ +--- +devices: + samplerate: 44100 + chunksize: 1024 + capture: + type: Stdin + channels: 2 + format: S16LE + playback: + type: Stdout + channels: 2 + format: S16LE \ No newline at end of file diff --git a/tests/testfiles/log.txt b/tests/testfiles/log.txt new file mode 100644 index 0000000..9285cd6 --- /dev/null +++ b/tests/testfiles/log.txt @@ -0,0 +1,2 @@ +Log message 1 +Log message 2 \ No newline at end of file From 27c1216a574ff27d90a94c857230bcb3084524a7 Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 23 Jan 2024 21:45:36 +0100 Subject: [PATCH 15/37] Don't write any .pyc during tests --- .github/workflows/build.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b97b404..a75586d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,23 +51,22 @@ jobs: - name: Install template render dependencies run: | python -m pip install --upgrade pip - pip install jinja2 PyYAML + python -m pip install jinja2 PyYAML - name: Render scripts from templates run: python -m release_automation.render_env_files - name: Install requirements - run: python3 -m pip install -r requirements.txt + run: python -m pip install -r requirements.txt - name: Set up pytest - run: python3 -m pip install pytest-aiohttp + run: python -m pip install pytest-aiohttp - name: Run python tests - run: python3 -m pytest + run: python -Bm pytest - name: Clean up run: | - rm -rf backend/__pycache__ rm -rf release_automation rm -rf tests From 63a666e598681d153098d8b311edfaa72948e5e7 Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 23 Jan 2024 21:51:11 +0100 Subject: [PATCH 16/37] Don't write any .pyc during tests --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a75586d..15e35bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,7 +54,7 @@ jobs: python -m pip install jinja2 PyYAML - name: Render scripts from templates - run: python -m release_automation.render_env_files + run: python -Bm release_automation.render_env_files - name: Install requirements run: python -m pip install -r requirements.txt From cf80bf79c3dd6889a3ff3a9f4750ba049634eb3f Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 23 Jan 2024 21:57:49 +0100 Subject: [PATCH 17/37] Mock library vesion --- tests/test_basic_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 77cd1df..27ec31c 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -82,9 +82,10 @@ def mock_app(): client.status.processing_load = MagicMock(side_effect=[0.5]) client.config = MagicMock() client.config.active = MagicMock(side_effect=[SAMPLE_CONFIG]) + client.versions = MagicMock() + client.versions.library = MagicMock(side_effect=["1.2.3"]) with patch("camilladsp.CamillaClient", client_constructor): app = main.build_app(server_config) - app["STATUSCACHE"]["py_cdsp_version"] = "1.2.3" yield app From c3e8b422f6e2e225bd6c489440f397c72d5f6053 Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 23 Jan 2024 22:33:27 +0100 Subject: [PATCH 18/37] test upload and delete --- tests/test_basic_api.py | 58 +++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 27ec31c..4b9d7ec 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -1,11 +1,12 @@ import json import pytest from unittest.mock import MagicMock, patch -import asyncio import pytest -from aiohttp import web +from aiohttp import web, FormData import os import yaml +import random +import string import main from backend import views @@ -66,15 +67,13 @@ def mock_app(): client.general.state = MagicMock( side_effect=[camilladsp.camilladsp.ProcessingState.RUNNING] ) - client.general.list_capture_devices = MagicMock(side_effect=[[ - ["hw:Aaaa,0,0", "Dev A"], - ["hw:Bbbb,0,0", "Dev B"] - ]]) - client.general.list_playback_devices = MagicMock(side_effect=[[ - ["hw:Cccc,0,0", "Dev C"], - ["hw:Dddd,0,0", "Dev D"] - ]]) - client.general.supported_device_types = MagicMock(side_effect = [["Alsa", "Wasapi"]]) + client.general.list_capture_devices = MagicMock( + side_effect=[[["hw:Aaaa,0,0", "Dev A"], ["hw:Bbbb,0,0", "Dev B"]]] + ) + client.general.list_playback_devices = MagicMock( + side_effect=[[["hw:Cccc,0,0", "Dev C"], ["hw:Dddd,0,0", "Dev D"]]] + ) + client.general.supported_device_types = MagicMock(side_effect=[["Alsa", "Wasapi"]]) client.status = MagicMock() client.status.rate_adjust = MagicMock(side_effect=[1.01]) client.status.buffer_level = MagicMock(side_effect=[1234]) @@ -157,3 +156,40 @@ async def test_all_get_endpoints_ok(server, endpoint, parameters): else: resp = await server.get(endpoint) assert resp.status == 200 + + +@pytest.mark.parametrize( + "upload, delete, getfile", + [ + ("/api/uploadconfigs", "/api/deleteconfigs", "/config/"), + ("/api/uploadcoeffs", "/api/deletecoeffs", "/coeff/"), + ], +) +@pytest.mark.asyncio +async def test_upload_and_delete(server, upload, delete, getfile): + filename = ''.join(random.choice(string.ascii_lowercase) for i in range(10)) + filedata = ''.join(random.choice(string.ascii_lowercase) for i in range(10)) + + # try to get a file that does not exist + resp = await server.get(getfile + filename) + assert resp.status == 404 + + # generate and upload a file + data = FormData() + data.add_field("file0", filedata.encode(), filename=filename) + resp = await server.post(upload, data=data) + assert resp.status == 200 + + # fetch the file, check the content + resp = await server.get(getfile + filename) + assert resp.status == 200 + response_data = await resp.read() + assert response_data == filedata.encode() + + # delete the file + resp = await server.post(delete, json=[filename]) + assert resp.status == 200 + + # try to download the deleted file + resp = await server.get(getfile + filename) + assert resp.status == 404 From 695cf7e2b076b3daf56146efbe63de0b0a4cdc04 Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 23 Jan 2024 22:34:18 +0100 Subject: [PATCH 19/37] format --- tests/test_basic_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 4b9d7ec..96dc985 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -167,8 +167,8 @@ async def test_all_get_endpoints_ok(server, endpoint, parameters): ) @pytest.mark.asyncio async def test_upload_and_delete(server, upload, delete, getfile): - filename = ''.join(random.choice(string.ascii_lowercase) for i in range(10)) - filedata = ''.join(random.choice(string.ascii_lowercase) for i in range(10)) + filename = "".join(random.choice(string.ascii_lowercase) for i in range(10)) + filedata = "".join(random.choice(string.ascii_lowercase) for i in range(10)) # try to get a file that does not exist resp = await server.get(getfile + filename) From 72a5b9bb3dbcd3654b44f52ba592afc01830027d Mon Sep 17 00:00:00 2001 From: Henrik Date: Wed, 24 Jan 2024 21:53:17 +0100 Subject: [PATCH 20/37] Add tests for config handling --- tests/test_basic_api.py | 89 ++++++++++++++++++-------- tests/testfiles/config2.yml | 12 ++++ tests/testfiles/statefile_template.yml | 14 ++++ 3 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 tests/testfiles/config2.yml create mode 100644 tests/testfiles/statefile_template.yml diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 96dc985..1bed69f 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -14,6 +14,10 @@ TESTFILE_DIR = os.path.join(os.path.dirname(__file__), "testfiles") SAMPLE_CONFIG = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "config.yml"))) +statefile_data = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "statefile_template.yml"))) +statefile_data["config_path"] = os.path.join(TESTFILE_DIR, "config2.yml") +with open(os.path.join(TESTFILE_DIR, "statefile.yml"), "w") as f: + yaml.dump(statefile_data, f) server_config = { "camilla_host": "127.0.0.1", @@ -39,51 +43,62 @@ def mock_request(mock_app): request.app = mock_app yield request - @pytest.fixture -def mock_app(): +def mock_camillaclient(): client = MagicMock() client_constructor = MagicMock(return_value=client) + client_constructor._client = client client.volume = MagicMock() - client.volume.main = MagicMock(side_effect=[-20.0]) + client.volume.main = MagicMock(return_value=-20.0) client.mute = MagicMock() - client.mute.main = MagicMock(side_effect=[False]) + client.mute.main = MagicMock(return_value=False) client.levels = MagicMock - client.levels.capture_peak = MagicMock(side_effect=[[-2.0, -3.0]]) - client.levels.playback_peak = MagicMock(side_effect=[[-2.5, -3.5]]) + client.levels.capture_peak = MagicMock(return_value=[-2.0, -3.0]) + client.levels.playback_peak = MagicMock(return_value=[-2.5, -3.5]) client.levels.levels = MagicMock( - side_effect=[ - { - "capture_rms": [-5.0, -6.0], - "capture_peak": [-2.0, -3.0], - "playback_rms": [-7.0, -8.0], - "playback_peak": [-3.0, -4.0], - } - ] + return_value={ + "capture_rms": [-5.0, -6.0], + "capture_peak": [-2.0, -3.0], + "playback_rms": [-7.0, -8.0], + "playback_peak": [-3.0, -4.0], + } ) client.rate = MagicMock() - client.rate.capture = MagicMock(side_effect=[44100]) + client.rate.capture = MagicMock(return_value=44100) client.general = MagicMock() client.general.state = MagicMock( - side_effect=[camilladsp.camilladsp.ProcessingState.RUNNING] + return_value=camilladsp.camilladsp.ProcessingState.RUNNING ) client.general.list_capture_devices = MagicMock( - side_effect=[[["hw:Aaaa,0,0", "Dev A"], ["hw:Bbbb,0,0", "Dev B"]]] + return_value=[["hw:Aaaa,0,0", "Dev A"], ["hw:Bbbb,0,0", "Dev B"]] ) client.general.list_playback_devices = MagicMock( - side_effect=[[["hw:Cccc,0,0", "Dev C"], ["hw:Dddd,0,0", "Dev D"]]] + return_value=[["hw:Cccc,0,0", "Dev C"], ["hw:Dddd,0,0", "Dev D"]] ) - client.general.supported_device_types = MagicMock(side_effect=[["Alsa", "Wasapi"]]) + client.general.supported_device_types = MagicMock(return_value=["Alsa", "Wasapi"]) client.status = MagicMock() - client.status.rate_adjust = MagicMock(side_effect=[1.01]) - client.status.buffer_level = MagicMock(side_effect=[1234]) - client.status.clipped_samples = MagicMock(side_effect=[12]) - client.status.processing_load = MagicMock(side_effect=[0.5]) + client.status.rate_adjust = MagicMock(return_value=1.01) + client.status.buffer_level = MagicMock(return_value=1234) + client.status.clipped_samples = MagicMock(return_value=12) + client.status.processing_load = MagicMock(return_value=0.5) client.config = MagicMock() - client.config.active = MagicMock(side_effect=[SAMPLE_CONFIG]) + client.config.active = MagicMock(return_value=SAMPLE_CONFIG) + client.config.file_path = MagicMock(return_value = os.path.join(TESTFILE_DIR, "config.yml")) client.versions = MagicMock() - client.versions.library = MagicMock(side_effect=["1.2.3"]) - with patch("camilladsp.CamillaClient", client_constructor): + client.versions.library = MagicMock(return_value="1.2.3") + yield client_constructor + +@pytest.fixture +def mock_app(mock_camillaclient): + with patch("camilladsp.CamillaClient", mock_camillaclient): + app = main.build_app(server_config) + yield app + +@pytest.fixture +def mock_offline_app(mock_camillaclient): + mock_camillaclient._client.config.file_path = MagicMock(return_value = None) + mock_camillaclient._client.general.state = MagicMock(side_effect = camilladsp.CamillaError) + with patch("camilladsp.CamillaClient", mock_camillaclient): app = main.build_app(server_config) yield app @@ -92,6 +107,9 @@ def mock_app(): def server(event_loop, aiohttp_client, mock_app): return event_loop.run_until_complete(aiohttp_client(mock_app)) +@pytest.fixture +def offline_server(event_loop, aiohttp_client, mock_offline_app): + return event_loop.run_until_complete(aiohttp_client(mock_offline_app)) @pytest.mark.asyncio async def test_read_volume(mock_request): @@ -193,3 +211,22 @@ async def test_upload_and_delete(server, upload, delete, getfile): # try to download the deleted file resp = await server.get(getfile + filename) assert resp.status == 404 + + +@pytest.mark.asyncio +async def test_active_config_online(server): + resp = await server.get("/api/getactiveconfigfile") + assert resp.status == 200 + content = await resp.json() + print(content) + assert content["configFileName"] == "config.yml" + assert content["config"]["devices"]["samplerate"] == 44100 + +@pytest.mark.asyncio +async def test_active_config_offline(offline_server): + resp = await offline_server.get("/api/getactiveconfigfile") + assert resp.status == 200 + content = await resp.json() + print(content) + assert content["configFileName"] == "config2.yml" + assert content["config"]["devices"]["samplerate"] == 48000 \ No newline at end of file diff --git a/tests/testfiles/config2.yml b/tests/testfiles/config2.yml new file mode 100644 index 0000000..e76e953 --- /dev/null +++ b/tests/testfiles/config2.yml @@ -0,0 +1,12 @@ +--- +devices: + samplerate: 48000 + chunksize: 1024 + capture: + type: Stdin + channels: 2 + format: S16LE + playback: + type: Stdout + channels: 2 + format: S16LE \ No newline at end of file diff --git a/tests/testfiles/statefile_template.yml b/tests/testfiles/statefile_template.yml new file mode 100644 index 0000000..562b1cb --- /dev/null +++ b/tests/testfiles/statefile_template.yml @@ -0,0 +1,14 @@ +--- +config_path: config2.yml +mute: + - false + - false + - false + - false + - false +volume: + - 0.0 + - 0.0 + - 0.0 + - 0.0 + - 0.0 \ No newline at end of file From c587df1139ce49786ad6c6c054fb87b25c7e30ce Mon Sep 17 00:00:00 2001 From: Henrik Date: Wed, 24 Jan 2024 22:00:45 +0100 Subject: [PATCH 21/37] Improver statefile mocking --- tests/test_basic_api.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 1bed69f..988ca08 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -14,10 +14,13 @@ TESTFILE_DIR = os.path.join(os.path.dirname(__file__), "testfiles") SAMPLE_CONFIG = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "config.yml"))) -statefile_data = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "statefile_template.yml"))) -statefile_data["config_path"] = os.path.join(TESTFILE_DIR, "config2.yml") -with open(os.path.join(TESTFILE_DIR, "statefile.yml"), "w") as f: - yaml.dump(statefile_data, f) +@pytest.fixture +def statefile(): + statefile_data = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "statefile_template.yml"))) + statefile_data["config_path"] = os.path.join(TESTFILE_DIR, statefile_data["config_path"]) + with open(os.path.join(TESTFILE_DIR, "statefile.yml"), "w") as f: + yaml.dump(statefile_data, f) + server_config = { "camilla_host": "127.0.0.1", @@ -44,7 +47,7 @@ def mock_request(mock_app): yield request @pytest.fixture -def mock_camillaclient(): +def mock_camillaclient(statefile): client = MagicMock() client_constructor = MagicMock(return_value=client) client_constructor._client = client From cf99708a9efce08d5af1500fbfe847e8fd482e0d Mon Sep 17 00:00:00 2001 From: Henrik Date: Wed, 24 Jan 2024 22:08:54 +0100 Subject: [PATCH 22/37] Use constants, format --- tests/test_basic_api.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 988ca08..0398dd9 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -13,12 +13,20 @@ import camilladsp TESTFILE_DIR = os.path.join(os.path.dirname(__file__), "testfiles") -SAMPLE_CONFIG = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "config.yml"))) +SAMPLE_CONFIG_PATH = os.path.join(TESTFILE_DIR, "config.yml") +STATEFILE_PATH = os.path.join(TESTFILE_DIR, "statefile.yml") +STATEFILE_TEMPLATE_PATH = os.path.join(TESTFILE_DIR, "statefile_template.yml") +LOGFILE_PATH = os.path.join(TESTFILE_DIR, "log.txt") +SAMPLE_CONFIG = yaml.safe_load(open(SAMPLE_CONFIG_PATH)) + + @pytest.fixture def statefile(): - statefile_data = yaml.safe_load(open(os.path.join(TESTFILE_DIR, "statefile_template.yml"))) - statefile_data["config_path"] = os.path.join(TESTFILE_DIR, statefile_data["config_path"]) - with open(os.path.join(TESTFILE_DIR, "statefile.yml"), "w") as f: + statefile_data = yaml.safe_load(open(STATEFILE_TEMPLATE_PATH)) + statefile_data["config_path"] = os.path.join( + TESTFILE_DIR, statefile_data["config_path"] + ) + with open(STATEFILE_PATH, "w") as f: yaml.dump(statefile_data, f) @@ -29,9 +37,9 @@ def statefile(): "port": 5005, "config_dir": TESTFILE_DIR, "coeff_dir": TESTFILE_DIR, - "default_config": os.path.join(TESTFILE_DIR, "config.yml"), - "statefile_path": os.path.join(TESTFILE_DIR, "statefile.yml"), - "log_file": os.path.join(TESTFILE_DIR, "log.txt"), + "default_config": SAMPLE_CONFIG_PATH, + "statefile_path": STATEFILE_PATH, + "log_file": LOGFILE_PATH, "on_set_active_config": None, "on_get_active_config": None, "supported_capture_types": None, @@ -46,6 +54,7 @@ def mock_request(mock_app): request.app = mock_app yield request + @pytest.fixture def mock_camillaclient(statefile): client = MagicMock() @@ -86,21 +95,25 @@ def mock_camillaclient(statefile): client.status.processing_load = MagicMock(return_value=0.5) client.config = MagicMock() client.config.active = MagicMock(return_value=SAMPLE_CONFIG) - client.config.file_path = MagicMock(return_value = os.path.join(TESTFILE_DIR, "config.yml")) + client.config.file_path = MagicMock(return_value=SAMPLE_CONFIG_PATH) client.versions = MagicMock() client.versions.library = MagicMock(return_value="1.2.3") yield client_constructor + @pytest.fixture def mock_app(mock_camillaclient): with patch("camilladsp.CamillaClient", mock_camillaclient): app = main.build_app(server_config) yield app + @pytest.fixture def mock_offline_app(mock_camillaclient): - mock_camillaclient._client.config.file_path = MagicMock(return_value = None) - mock_camillaclient._client.general.state = MagicMock(side_effect = camilladsp.CamillaError) + mock_camillaclient._client.config.file_path = MagicMock(return_value=None) + mock_camillaclient._client.general.state = MagicMock( + side_effect=camilladsp.CamillaError + ) with patch("camilladsp.CamillaClient", mock_camillaclient): app = main.build_app(server_config) yield app @@ -110,10 +123,12 @@ def mock_offline_app(mock_camillaclient): def server(event_loop, aiohttp_client, mock_app): return event_loop.run_until_complete(aiohttp_client(mock_app)) + @pytest.fixture def offline_server(event_loop, aiohttp_client, mock_offline_app): return event_loop.run_until_complete(aiohttp_client(mock_offline_app)) + @pytest.mark.asyncio async def test_read_volume(mock_request): mock_request.match_info = {"name": "volume"} @@ -225,6 +240,7 @@ async def test_active_config_online(server): assert content["configFileName"] == "config.yml" assert content["config"]["devices"]["samplerate"] == 44100 + @pytest.mark.asyncio async def test_active_config_offline(offline_server): resp = await offline_server.get("/api/getactiveconfigfile") @@ -232,4 +248,4 @@ async def test_active_config_offline(offline_server): content = await resp.json() print(content) assert content["configFileName"] == "config2.yml" - assert content["config"]["devices"]["samplerate"] == 48000 \ No newline at end of file + assert content["config"]["devices"]["samplerate"] == 48000 From eb65dac2bbec4222c6422ee858e37c4370217bcb Mon Sep 17 00:00:00 2001 From: Henrik Date: Sun, 4 Feb 2024 22:37:27 +0100 Subject: [PATCH 23/37] WIP migrate old camilladsp configs on import --- backend/legacy_config_import.py | 65 ++++++++++++++++++++++++++++++ tests/test_legacy_config.py | 70 +++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 backend/legacy_config_import.py create mode 100644 tests/test_legacy_config.py diff --git a/backend/legacy_config_import.py b/backend/legacy_config_import.py new file mode 100644 index 0000000..1855543 --- /dev/null +++ b/backend/legacy_config_import.py @@ -0,0 +1,65 @@ +def _remove_volume_filters(config): + """ + Remove any Volume filter + """ + pass + +def _modify_loundness_filters(config): + """ + Modify Loudness filters + """ + pass + +def _modify_resampler(config): + """ + Update the resampler config + """ + if "enable_resampling" in config["devices"]: + if config["devices"]["enable_resampling"]: + # TODO map the easy presets, skip the free? + pass + else: + config["devices"]["resampler"] = None + del config["devices"]["enable_resampling"] + +def _modify_devices(config): + """ + Update the options in the devices section + """ + # New logic for setting sample format + dev = config["devices"]["capture"] + _modify_coreaudio_device(dev) + dev = config["devices"]["playback"] + _modify_coreaudio_device(dev) + + # Resampler + _modify_resampler(config) + + # Basic options + + +def _modify_coreaudio_device(dev): + if dev["type"] == "CoreAudio": + if "change_format" in dev: + if not dev["change_format"]: + dev["format"] = None + del dev["change_format"] + else: + dev["format"] = None + +def _modify_dither(config): + """ + Update Dither filters + """ + pass + +def migrate_legacy_config(config): + """ + Modifies an older config file to the latest format. + The modifications are done in-place. + """ + _remove_volume_filters(config) + _modify_loundness_filters(config) + _modify_dither(config) + _modify_devices(config) + diff --git a/tests/test_legacy_config.py b/tests/test_legacy_config.py new file mode 100644 index 0000000..49fc2de --- /dev/null +++ b/tests/test_legacy_config.py @@ -0,0 +1,70 @@ +import pytest + +from backend.legacy_config_import import _modify_devices + +@pytest.fixture +def basic_config(): + config = { + "devices": { + "samplerate": 96000, + "chunksize": 2048, + "queuelimit": 4, + "silence_threshold": -60, + "silence_timeout": 3.0, + "target_level": 500, + "adjust_period": 10, + "enable_rate_adjust": True, + "resampler_type": "BalancedAsync", + "enable_resampling": False, + "capture_samplerate": 44100, + "stop_on_rate_change": False, + "rate_measure_interval": 1.0, + "capture": { + "type": "Stdin", + "channels": 2, + "format": "S16LE" + }, + "playback": { + "type": "Stdout", + "channels": 2, + "format": "S32LE" + } + }, + "filters": {}, + "mixers": {}, + "pipeline": {} + } + yield config + + +def test_coreaudio_device(basic_config): + config = basic_config + config["devices"]["capture"] = { + "type": "CoreAudio", + "channels": 2, + "device": "Soundflower (2ch)", + "format": "S32LE", + "change_format": True + } + config["devices"]["playback"] = { + "type": "CoreAudio", + "channels": 2, + "device": "Built-in Output", + "format": "S32LE", + "exclusive": False, + "change_format": False + } + + _modify_devices(config) + capture = config["devices"]["capture"] + playback = config["devices"]["playback"] + assert "change_format" not in capture + assert "change_format" not in playback + assert capture["format"] == "S32LE" + assert playback["format"] == None + +def test_disabled_resampling(basic_config): + _modify_devices(basic_config) + assert "enable_resampling" not in basic_config["devices"] + assert basic_config["devices"]["resampler"] == None + From baa9dfdf3544da6eeb7b2fdd8ad685d97401e91d Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 6 Feb 2024 21:37:00 +0100 Subject: [PATCH 24/37] Add the missing migrations --- backend/legacy_config_import.py | 80 ++++++++++++++++++++--- backend/views.py | 5 +- tests/test_legacy_config.py | 111 +++++++++++++++++++++++++++----- 3 files changed, 168 insertions(+), 28 deletions(-) diff --git a/backend/legacy_config_import.py b/backend/legacy_config_import.py index 1855543..ac0a581 100644 --- a/backend/legacy_config_import.py +++ b/backend/legacy_config_import.py @@ -2,13 +2,34 @@ def _remove_volume_filters(config): """ Remove any Volume filter """ - pass + if "filters" in config: + volume_names = [] + for name, params in list(config["filters"].items()): + if params["type"] == "Volume": + volume_names.append(name) + del config["filters"][name] + + if "pipeline" in config: + for step in list(config["pipeline"]): + if step["type"] == "Filter": + step["names"] = [ + name for name in step["names"] if name not in volume_names + ] + if len(step["names"]) == 0: + config["pipeline"].remove(step) + def _modify_loundness_filters(config): """ Modify Loudness filters """ - pass + if "filters" in config: + for name, params in config["filters"].items(): + if params["type"] == "Loudness": + del params["parameters"]["ramp_time"] + params["parameters"]["fader"] = "Main" + params["parameters"]["attenuate_mid"] = False + def _modify_resampler(config): """ @@ -17,10 +38,42 @@ def _modify_resampler(config): if "enable_resampling" in config["devices"]: if config["devices"]["enable_resampling"]: # TODO map the easy presets, skip the free? - pass + if config["devices"]["resampler_type"] == "Synchronous": + config["devices"]["resampler"] = {"type": "Synchronous"} + elif config["devices"]["resampler_type"] == "FastAsync": + config["devices"]["resampler"] = { + "type": "AsyncSinc", + "profile": "Fast", + } + elif config["devices"]["resampler_type"] == "BalancedAsync": + config["devices"]["resampler"] = { + "type": "AsyncSinc", + "profile": "Balanced", + } + elif config["devices"]["resampler_type"] == "AccurateAsync": + config["devices"]["resampler"] = { + "type": "AsyncSinc", + "profile": "Accurate", + } + elif isinstance(config["devices"]["resampler_type"], dict): + old_resampler = config["devices"]["resampler_type"] + if "FreeAsync" in old_resampler: + params = old_resampler["FreeAsync"] + new_resampler = { + "type": "AsyncSinc", + "sinc_len": params["sinc_len"], + "oversampling_factor": params["oversampling_ratio"], + "interpolation": params["interpolation"], + "window": params["window"], + "f_cutoff": params["f_cutoff"], + } + config["devices"]["resampler"] = new_resampler else: config["devices"]["resampler"] = None del config["devices"]["enable_resampling"] + if "resampler_type" in config["devices"]: + del config["devices"]["resampler_type"] + def _modify_devices(config): """ @@ -31,12 +84,10 @@ def _modify_devices(config): _modify_coreaudio_device(dev) dev = config["devices"]["playback"] _modify_coreaudio_device(dev) - + # Resampler _modify_resampler(config) - # Basic options - def _modify_coreaudio_device(dev): if dev["type"] == "CoreAudio": @@ -47,11 +98,21 @@ def _modify_coreaudio_device(dev): else: dev["format"] = None + def _modify_dither(config): """ - Update Dither filters + Update Dither filters, some names have changed. + Uniform -> Flat + Simple -> Highpass """ - pass + if "filters" in config: + for _name, params in config["filters"].items(): + if params["type"] == "Dither": + if params["parameters"]["type"] == "Uniform": + params["parameters"]["type"] = "Flat" + elif params["parameters"]["type"] == "Simple": + params["parameters"]["type"] = "Highpass" + def migrate_legacy_config(config): """ @@ -61,5 +122,4 @@ def migrate_legacy_config(config): _remove_volume_filters(config) _modify_loundness_filters(config) _modify_dither(config) - _modify_devices(config) - + _modify_devices(config) diff --git a/backend/views.py b/backend/views.py index c68e06e..a0cc21b 100644 --- a/backend/views.py +++ b/backend/views.py @@ -18,6 +18,7 @@ from .filters import defaults_for_filter, filter_plot_options, pipeline_step_plot_options from .settings import get_gui_config_or_defaults from .convolver_config_import import ConvolverConfig +from .legacy_config_import import migrate_legacy_config OFFLINE_CACHE = { "cdsp_status": "Offline", @@ -380,11 +381,13 @@ async def parse_and_validate_yml_config_to_json(request): async def yaml_to_json(request): """ - Parse a yaml string and return seralized as json. + Parse a yaml string and return serialized as json. This could also be just a partial config. + The config is migrated from older camilladsp versions if needed. """ config_yaml = await request.text() loaded = yaml.safe_load(config_yaml) + migrate_legacy_config(loaded) return web.json_response(loaded) diff --git a/tests/test_legacy_config.py b/tests/test_legacy_config.py index 49fc2de..9a09fb3 100644 --- a/tests/test_legacy_config.py +++ b/tests/test_legacy_config.py @@ -1,6 +1,14 @@ import pytest -from backend.legacy_config_import import _modify_devices +from backend.legacy_config_import import ( + _modify_devices, + _remove_volume_filters, + _modify_loundness_filters, + _modify_dither, + migrate_legacy_config, +) +from camilladsp_plot.validate_config import CamillaValidator + @pytest.fixture def basic_config(): @@ -19,32 +27,44 @@ def basic_config(): "capture_samplerate": 44100, "stop_on_rate_change": False, "rate_measure_interval": 1.0, - "capture": { - "type": "Stdin", - "channels": 2, - "format": "S16LE" + "capture": {"type": "Stdin", "channels": 2, "format": "S16LE"}, + "playback": {"type": "Stdout", "channels": 2, "format": "S32LE"}, + }, + "filters": { + "vol": {"type": "Volume", "parameters": {"ramp_time": 200}}, + "hp_80": { + "type": "Biquad", + "parameters": {"type": "Highpass", "freq": 80, "q": 0.5}, + }, + "loudness": { + "type": "Loudness", + "parameters": { + "ramp_time": 200.0, + "reference_level": -25.0, + "high_boost": 7.0, + "low_boost": 7.0, }, - "playback": { - "type": "Stdout", - "channels": 2, - "format": "S32LE" - } + }, + "dither": {"type": "Dither", "parameters": {"type": "Simple", "bits": 16}}, }, - "filters": {}, "mixers": {}, - "pipeline": {} + "pipeline": [ + {"type": "Filter", "channel": 0, "names": ["vol", "hp_80"]}, + {"type": "Filter", "channel": 1, "names": ["vol"]}, + ], } yield config - + def test_coreaudio_device(basic_config): config = basic_config + # Insert CoreAudio capture and playback devices config["devices"]["capture"] = { "type": "CoreAudio", "channels": 2, "device": "Soundflower (2ch)", "format": "S32LE", - "change_format": True + "change_format": True, } config["devices"]["playback"] = { "type": "CoreAudio", @@ -52,19 +72,76 @@ def test_coreaudio_device(basic_config): "device": "Built-in Output", "format": "S32LE", "exclusive": False, - "change_format": False + "change_format": False, } - _modify_devices(config) capture = config["devices"]["capture"] - playback = config["devices"]["playback"] + playback = config["devices"]["playback"] assert "change_format" not in capture assert "change_format" not in playback assert capture["format"] == "S32LE" assert playback["format"] == None + def test_disabled_resampling(basic_config): _modify_devices(basic_config) assert "enable_resampling" not in basic_config["devices"] assert basic_config["devices"]["resampler"] == None + +def test_removed_volume_filters(basic_config): + _remove_volume_filters(basic_config) + assert "vol" not in basic_config["filters"] + assert len(basic_config["pipeline"]) == 1 + assert basic_config["pipeline"][0]["names"] == ["hp_80"] + + +def test_update_loudness_filters(basic_config): + _modify_loundness_filters(basic_config) + params = basic_config["filters"]["loudness"]["parameters"] + assert "ramp_time" not in params + assert params["fader"] == "Main" + assert params["attenuate_mid"] == False + + +def test_modify_dither(basic_config): + _modify_dither(basic_config) + params = basic_config["filters"]["dither"]["parameters"] + assert params["type"] == "Highpass" + + +def test_free_resampler(basic_config): + basic_config["devices"]["resampler_type"] = { + "FreeAsync": { + "f_cutoff": 0.9, + "sinc_len": 128, + "window": "Hann2", + "oversampling_ratio": 64, + "interpolation": "Cubic", + } + } + basic_config["devices"]["enable_resampling"] = True + _modify_devices(basic_config) + assert "enable_resampling" not in basic_config["devices"] + assert basic_config["devices"]["resampler"] == { + "type": "AsyncSinc", + "f_cutoff": 0.9, + "sinc_len": 128, + "window": "Hann2", + "oversampling_factor": 64, + "interpolation": "Cubic", + } + + +def test_schema_validation(basic_config): + # verify that the test config is not yet valid + validator = CamillaValidator() + validator.validate_config(basic_config) + errors = validator.get_errors() + assert len(errors) > 0 + + # migrate and validate + migrate_legacy_config(basic_config) + validator.validate_config(basic_config) + errors = validator.get_errors() + assert len(errors) == 0 From c3dd16c1e9b47f210d8d827d08da0d57ba4f4e76 Mon Sep 17 00:00:00 2001 From: Henrik Date: Tue, 6 Feb 2024 22:48:29 +0100 Subject: [PATCH 25/37] WIP eqapo import --- backend/eqapo_config_import.py | 319 ++++++++++++++++++++++++++++++ tests/test_eqapo_config_import.py | 41 ++++ 2 files changed, 360 insertions(+) create mode 100644 backend/eqapo_config_import.py create mode 100644 tests/test_eqapo_config_import.py diff --git a/backend/eqapo_config_import.py b/backend/eqapo_config_import.py new file mode 100644 index 0000000..494a5c1 --- /dev/null +++ b/backend/eqapo_config_import.py @@ -0,0 +1,319 @@ +from copy import copy, deepcopy + +class EqAPO(): + filter_types = { + "PK": "Peaking", + "PEQ": "Peaking", + "HP": "Highpass", + "HPQ": "Highpass", + "LP": "Lowpass", + "LPQ": "Lowpass", + "BP": "Bandpass", + "NO": "Notch", + "LS": "Lowshelf", + "LSC": "Lowshelf", + "HS": "Highshelf", + "HSC": "Highshelf", + "IIR": None, #TODO + } + # TODO + # add support for + # HSC x dB: High-shelf filter x dB per oct. + # LSC x dB: Low-shelf filter x dB per oct. + # LS 6dB: Low-shelf filter 6 dB per octave, with corner freq. + # LS 12dB: Low-shelf filter 12 dB per octave, with corner freq. + # HS 6dB: High-shelf filter 6 dB per octave, with corner freq. + # HS 12dB: High-shelf filter 12 dB per octave, with corner freq. + + + # Label to channel number, assuming 7.1 + channel_map = { + "L": 0, + "R": 1, + "C": 2, + "LFE": 3, + "RL": 4, + "RR": 5, + "SL": 6, + "SR": 7 + } + + delay_units = { + "ms": "ms", + "samples": "samples" + } + + def __init__(self, config_text, nbr_channels): + self.lines = config_text.splitlines() + self.filters = {} + self.mixers = {} + self.name_index = { + "Filter": 1, + "Preamp": 1, + "Convolution": 1, + "Delay": 1, + "Copy": 1, + } + self.selected_channels = None + self.current_filterstep = { + "type": "Filter", + "names": [], + "description": "Default, all channels", + "channels": copy(self.selected_channels) + } + self.pipeline = [self.current_filterstep] + self.nbr_channels = nbr_channels + + def lookup_channel_index(self, label): + if label in self.channel_map: + channel = self.channel_map[label] + elif label.isnumeric(): + channel = int(label) - 1 + else: + # TODO handle this + channel = None + return channel + + # Parse a single command parameter + def parse_single_parameter(self, params): + # Inline expressions (ex: Fc `2*a`) are not supported + # TODO add a check for this. + if params[0] == "Fc": + nbr_tokens = 3 + assert params[2].lower() == "hz" + value = float(params[1]) + parsed = {"freq": value} + elif params[0] == "Q": + nbr_tokens = 2 + value = float(params[1]) + parsed = {"q": value} + elif params[0] == "Gain": + nbr_tokens = 3 + assert params[2].lower() == "db" + value = float(params[1]) + parsed = {"gain": value} + elif params[0] == "BW": + nbr_tokens = 3 + assert params[1].lower() == "oct" + value = float(params[2]) + parsed = {"bandwidth": value} + else: + print("Skipping unknown token:", params[0]) + return {}, params[1:] + return parsed, params[nbr_tokens:] + + # Parse the parameters for a command + def parse_filter_params(self, param_str): + params = param_str.split() + enabled = params[0] == "ON" + ftype = params[1] + ftype_c = self.filter_types[ftype] + param_dict = {"type": ftype_c} + tokens = params[2:] + while tokens: + p, tokens = self.parse_single_parameter(tokens) + param_dict.update(p) + return param_dict + + # Parse a Preamp command to a filter + def parse_gain(self, param_str): + params = param_str.split() + gain = float(params[0]) + if params[1].lower() != "db": + print("invalid preamp line:", param_str) + return + return { + "type": "Gain", + "parameters": { + "gain": gain, + "scale": "dB" + } + } + + # Parse a Delay command to a filter + def parse_delay(self, param_str): + params = param_str.split() + delay = float(params[0]) + unit = self.delay_units[params[1]] + return { + "type": "Delay", + "parameters": { + "delay": delay, + "unit": unit + } + } + + # Parse a Copy command into a Mixer + def parse_copy(self, param_str): + handled_channels = set() + mixer = { + "channels": { + "in": self.nbr_channels, + "out": self.nbr_channels, + }, + "mapping": [] + } + params = param_str.strip().split(" ") + for dest in params: + dest_ch, expr = dest.split("=") + dest_ch = self.lookup_channel_index(dest_ch) + handled_channels.add(dest_ch) + print("dest", dest_ch) + mapping = { + "dest": dest_ch, + "mute": False, + "sources": [] + } + mixer["mapping"].append(mapping) + sources = expr.split("+") + for source in sources: + if "*" in source: + gain_str, channel = source.split("*") + if gain_str.endswith("dB"): + gain = float(gain_str[:-2]) + scale = "dB" + else: + gain = float(gain_str) + scale = "linear" + elif source == "0.0": + # EqAPO supports setting channels to an arbitrary constant. + # Here only 0.0 is supported, as other values have no practical use. + channel = None + else: + gain = 0 + scale = "dB" + channel = source + if channel is not None: + channel = self.lookup_channel_index(channel) + # TODO make a mixer config + print("source", channel, gain, scale) + source = { + "channel": channel, + "gain": gain, + "inverted": False, + "scale": scale, + } + mapping["sources"].append(source) + for dest_ch in set(range(self.nbr_channels)) - handled_channels: + print("pass through", dest_ch) + mapping = { + "dest": dest_ch, + "mute": False, + "sources": [ + { + "channel": dest_ch, + "gain": 0.0, + "inverted": False, + "scale": "dB", + } + ] + } + mixer["mapping"].append(mapping) + return mixer + + + # Parse a single line + def parse_line(self, line): + if not line or line.startswith("#") or not ":" in line: + return + filtname = None + command_name, params = line.split(":", 1) + command = command_name.split()[0] + if command in ("Filter", "Convolution", "Preamp", "Delay"): + if command == "Filter": + filter = self.parse_filter_params(params) + filter = { + "type": "Biquad", + "parameters": filter + } + elif command == "Convolution": + filename = params.strip() + filter = { + "type": "Conv", + "parameters": { + "filename": filename, + "type": "wav" + } + } + elif command == "Preamp": + filter = self.parse_gain(params) + elif command == "Delay": + filter = self.parse_delay(params) + filter["description"] = line.strip() + filtname = f"{command}_{self.name_index[command]}" + self.name_index[command] += 1 + self.filters[filtname] = filter + self.pipeline[-1]["names"].append(filtname) + elif command == "Channel": + if params.strip() == "all": + self.selected_channels = None + else: + self.selected_channels = [self.lookup_channel_index(c) for c in params.strip().split(" ")] + new_filterstep = { + "type": "Filter", + "names": [], + "description": line.strip(), + "channels": copy(self.selected_channels), + } + self.pipeline.append(new_filterstep) + elif command == "Copy": + mixer = self.parse_copy(params) + mixer["description"] = line.strip() + mixername = f"{command}_{self.name_index[command]}" + self.name_index[command] += 1 + self.mixers[mixername] = mixer + step = { + "type": "Mixer", + "name": mixername, + } + self.pipeline.append(step) + step = { + "type": "Filter", + "names": [], + "description": "Continued after mixer", + "channels": copy(self.selected_channels) + } + self.pipeline.append(step) + elif command in ("Device", "Include", "Eval", "If", "ElseIf", "Else", "EndIf", "Stage", "GraphicEQ"): + print("Skipping unsupported command:", command) + + def postprocess(self): + for idx, step in enumerate(list(self.pipeline)): + if step["type"] == "Filter" and len(step["names"]) == 0: + self.pipeline.pop(idx) + for _, mixer in self.mixers.items(): + for idx, dest in enumerate(list(mixer["mapping"])): + if len(dest["sources"]) == 0: + mixer["mapping"].pop(idx) + # Expand filter steps to all channels + pipeline = [] + for step in self.pipeline: + if step["type"] != "Filter": + pipeline.append(step) + else: + channels = step["channels"] + if channels is None: + channels = range(self.nbr_channels) + for channel in channels: + new_step = deepcopy(step) + new_step["channel"] = channel + del new_step["channels"] + pipeline.append(new_step) + self.pipeline = pipeline + + def build_config(self): + config = { + "filters": self.filters, + "mixers": self.mixers, + "pipeline": self.pipeline + } + return config + + def translate_file(self): + for idx, line in enumerate(self.lines): + self.parse_line(line) + self.postprocess() + config = self.build_config() + return config + + diff --git a/tests/test_eqapo_config_import.py b/tests/test_eqapo_config_import.py new file mode 100644 index 0000000..a136f10 --- /dev/null +++ b/tests/test_eqapo_config_import.py @@ -0,0 +1,41 @@ +import pytest + +from backend.eqapo_config_import import ( + EqAPO +) + +EXAMPLE = """ +Device: High Definition Audio Device Speakers; Benchmark +#All lines below will only be applied to the specified device and the benchmark application +Preamp: -6 db +Include: example.txt +Filter 1: ON PK Fc 50 Hz Gain -3.0 dB Q 10.00 +Filter 2: ON PEQ Fc 100 Hz Gain 1.0 dB BW Oct 0.167 + +Channel: L +#Additional preamp for left channel +Preamp: -5 dB +#Filters only for left channel +Include: demo.txt +Filter 1: ON LS Fc 300 Hz Gain 5.0 dB + +Channel: 2 C +#Filters for second(right) and center channel +Filter 1: ON HP Fc 30 Hz +Filter 2: ON LPQ Fc 10000 Hz Q 0.400 + +Device: Microphone +#From here, the lines only apply to microphone devices +Filter: ON NO Fc 50 Hz +""" +@pytest.fixture +def eqapo(): + converter = EqAPO(EXAMPLE, 2) + yield converter + + +def test_single_filter(eqapo): + line = "Filter 1: ON PK Fc 50 Hz Gain -3.0 dB Q 10.00" + filt = eqapo.parse_line(line) + assert filter == None + \ No newline at end of file From 7a94ad4a9c81c65e4c3763c8f6e0bb1299b088bb Mon Sep 17 00:00:00 2001 From: Henrik Date: Wed, 7 Feb 2024 21:28:13 +0100 Subject: [PATCH 26/37] Update eqapo translator, add tests --- backend/eqapo_config_import.py | 109 +++++++++--------- tests/test_eqapo_config_import.py | 178 ++++++++++++++++++++++++++++-- 2 files changed, 220 insertions(+), 67 deletions(-) diff --git a/backend/eqapo_config_import.py b/backend/eqapo_config_import.py index 494a5c1..072276b 100644 --- a/backend/eqapo_config_import.py +++ b/backend/eqapo_config_import.py @@ -1,6 +1,7 @@ from copy import copy, deepcopy -class EqAPO(): + +class EqAPO: filter_types = { "PK": "Peaking", "PEQ": "Peaking", @@ -14,10 +15,10 @@ class EqAPO(): "LSC": "Lowshelf", "HS": "Highshelf", "HSC": "Highshelf", - "IIR": None, #TODO + "IIR": None, # TODO } # TODO - # add support for + # add support for # HSC x dB: High-shelf filter x dB per oct. # LSC x dB: Low-shelf filter x dB per oct. # LS 6dB: Low-shelf filter 6 dB per octave, with corner freq. @@ -25,23 +26,16 @@ class EqAPO(): # HS 6dB: High-shelf filter 6 dB per octave, with corner freq. # HS 12dB: High-shelf filter 12 dB per octave, with corner freq. - - # Label to channel number, assuming 7.1 - channel_map = { - "L": 0, - "R": 1, - "C": 2, - "LFE": 3, - "RL": 4, - "RR": 5, - "SL": 6, - "SR": 7 + # Label to channel number + all_channel_maps = { + 1: {"C": 1}, + 2: {"L": 0, "R": 1}, + 4: {"L": 0, "R": 1, "RL": 2, "RR": 3}, + 6: {"L": 0, "R": 1, "C": 2, "LFE": 3, "RL": 4, "RR": 5}, + 8: {"L": 0, "R": 1, "C": 2, "LFE": 3, "RL": 4, "RR": 5, "SL": 6, "SR": 7} } - delay_units = { - "ms": "ms", - "samples": "samples" - } + delay_units = {"ms": "ms", "samples": "samples"} def __init__(self, config_text, nbr_channels): self.lines = config_text.splitlines() @@ -59,10 +53,11 @@ def __init__(self, config_text, nbr_channels): "type": "Filter", "names": [], "description": "Default, all channels", - "channels": copy(self.selected_channels) + "channels": copy(self.selected_channels), } self.pipeline = [self.current_filterstep] self.nbr_channels = nbr_channels + self.channel_map = self.all_channel_maps.get(nbr_channels, self.all_channel_maps[8]) def lookup_channel_index(self, label): if label in self.channel_map: @@ -107,7 +102,10 @@ def parse_filter_params(self, param_str): params = param_str.split() enabled = params[0] == "ON" ftype = params[1] - ftype_c = self.filter_types[ftype] + ftype_c = self.filter_types.get(ftype) + if not ftype_c: + print(f"Unsupported filter type '{ftype}'") + return None param_dict = {"type": ftype_c} tokens = params[2:] while tokens: @@ -122,26 +120,14 @@ def parse_gain(self, param_str): if params[1].lower() != "db": print("invalid preamp line:", param_str) return - return { - "type": "Gain", - "parameters": { - "gain": gain, - "scale": "dB" - } - } + return {"type": "Gain", "parameters": {"gain": gain, "scale": "dB"}} # Parse a Delay command to a filter def parse_delay(self, param_str): params = param_str.split() delay = float(params[0]) unit = self.delay_units[params[1]] - return { - "type": "Delay", - "parameters": { - "delay": delay, - "unit": unit - } - } + return {"type": "Delay", "parameters": {"delay": delay, "unit": unit}} # Parse a Copy command into a Mixer def parse_copy(self, param_str): @@ -151,7 +137,7 @@ def parse_copy(self, param_str): "in": self.nbr_channels, "out": self.nbr_channels, }, - "mapping": [] + "mapping": [], } params = param_str.strip().split(" ") for dest in params: @@ -159,11 +145,7 @@ def parse_copy(self, param_str): dest_ch = self.lookup_channel_index(dest_ch) handled_channels.add(dest_ch) print("dest", dest_ch) - mapping = { - "dest": dest_ch, - "mute": False, - "sources": [] - } + mapping = {"dest": dest_ch, "mute": False, "sources": []} mixer["mapping"].append(mapping) sources = expr.split("+") for source in sources: @@ -206,12 +188,11 @@ def parse_copy(self, param_str): "inverted": False, "scale": "dB", } - ] + ], } mixer["mapping"].append(mapping) return mixer - # Parse a single line def parse_line(self, line): if not line or line.startswith("#") or not ":" in line: @@ -219,21 +200,18 @@ def parse_line(self, line): filtname = None command_name, params = line.split(":", 1) command = command_name.split()[0] + print("Parse command:", command) if command in ("Filter", "Convolution", "Preamp", "Delay"): if command == "Filter": - filter = self.parse_filter_params(params) - filter = { - "type": "Biquad", - "parameters": filter - } + filterparams = self.parse_filter_params(params) + if not filterparams: + return + filter = {"type": "Biquad", "parameters": filterparams} elif command == "Convolution": filename = params.strip() filter = { "type": "Conv", - "parameters": { - "filename": filename, - "type": "wav" - } + "parameters": {"filename": filename, "type": "wav"}, } elif command == "Preamp": filter = self.parse_gain(params) @@ -248,7 +226,9 @@ def parse_line(self, line): if params.strip() == "all": self.selected_channels = None else: - self.selected_channels = [self.lookup_channel_index(c) for c in params.strip().split(" ")] + self.selected_channels = [ + self.lookup_channel_index(c) for c in params.strip().split(" ") + ] new_filterstep = { "type": "Filter", "names": [], @@ -271,16 +251,29 @@ def parse_line(self, line): "type": "Filter", "names": [], "description": "Continued after mixer", - "channels": copy(self.selected_channels) + "channels": copy(self.selected_channels), } self.pipeline.append(step) - elif command in ("Device", "Include", "Eval", "If", "ElseIf", "Else", "EndIf", "Stage", "GraphicEQ"): - print("Skipping unsupported command:", command) + elif command in ( + "Device", + "Include", + "Eval", + "If", + "ElseIf", + "Else", + "EndIf", + "Stage", + "GraphicEQ", + ): + print(f"Command '{command}' is not supported, skipping.") + else: + print(f"Skipping unrecognized command '{command}'") def postprocess(self): for idx, step in enumerate(list(self.pipeline)): if step["type"] == "Filter" and len(step["names"]) == 0: - self.pipeline.pop(idx) + print("remove", step) + self.pipeline.remove(step) for _, mixer in self.mixers.items(): for idx, dest in enumerate(list(mixer["mapping"])): if len(dest["sources"]) == 0: @@ -305,7 +298,7 @@ def build_config(self): config = { "filters": self.filters, "mixers": self.mixers, - "pipeline": self.pipeline + "pipeline": self.pipeline, } return config @@ -315,5 +308,3 @@ def translate_file(self): self.postprocess() config = self.build_config() return config - - diff --git a/tests/test_eqapo_config_import.py b/tests/test_eqapo_config_import.py index a136f10..a57e580 100644 --- a/tests/test_eqapo_config_import.py +++ b/tests/test_eqapo_config_import.py @@ -1,8 +1,6 @@ import pytest -from backend.eqapo_config_import import ( - EqAPO -) +from backend.eqapo_config_import import EqAPO EXAMPLE = """ Device: High Definition Audio Device Speakers; Benchmark @@ -28,14 +26,178 @@ #From here, the lines only apply to microphone devices Filter: ON NO Fc 50 Hz """ + + @pytest.fixture def eqapo(): converter = EqAPO(EXAMPLE, 2) yield converter -def test_single_filter(eqapo): - line = "Filter 1: ON PK Fc 50 Hz Gain -3.0 dB Q 10.00" - filt = eqapo.parse_line(line) - assert filter == None - \ No newline at end of file +PK_EQAPO = "Filter 1: ON PK Fc 50 Hz Gain -3.0 dB Q 10.00" +PK_CDSP = {"freq": 50.0, "gain": -3.0, "q": 10.0, "type": "Peaking"} + +PEQ_EQAPO = "Filter 2: ON PEQ Fc 100 Hz Gain 1.0 dB BW Oct 0.167" +PEQ_CDSP = {"freq": 100.0, "gain": 1.0, "bandwidth": 0.167, "type": "Peaking"} + +@pytest.mark.parametrize( + "filterline, expected_params", + [ + (PK_EQAPO, PK_CDSP), + (PEQ_EQAPO, PEQ_CDSP) + ], +) +def test_single_filter(eqapo, filterline, expected_params): + eqapo.parse_line(filterline) + name, filt = next(iter(eqapo.filters.items())) + assert filt["parameters"] == expected_params + assert name == "Filter_1" + + +SIMPLE_CONV_EQAPO = """ +Channel: L +Convolution: L.wav +Channel: R +Convolution: R.wav +""" + +SIMPLE_CONV_CDSP = { + "filters": { + "Convolution_1": { + "type": "Conv", + "parameters": {"filename": "L.wav", "type": "wav"}, + "description": "Convolution: L.wav", + }, + "Convolution_2": { + "type": "Conv", + "parameters": {"filename": "R.wav", "type": "wav"}, + "description": "Convolution: R.wav", + }, + }, + "mixers": {}, + "pipeline": [ + { + "type": "Filter", + "names": ["Convolution_1"], + "description": "Channel: L", + "channel": 0, + }, + { + "type": "Filter", + "names": ["Convolution_2"], + "description": "Channel: R", + "channel": 1, + }, + ], +} + + +def test_simple_conv(): + converter = EqAPO(SIMPLE_CONV_EQAPO, 2) + converter.translate_file() + conf = converter.build_config() + assert conf == SIMPLE_CONV_CDSP + + +CROSSOVER_EQAPO = """ +Copy: RL=L RR=R +Channel: L R +Filter 1: ON LP Fc 2000 Hz +Channel: RL RR +Filter 2: ON HP Fc 2000 Hz +""" + +CROSSOVER_CDSP = { + "filters": { + "Filter_1": { + "type": "Biquad", + "parameters": {"type": "Lowpass", "freq": 2000.0}, + "description": "Filter 1: ON LP Fc 2000 Hz", + }, + "Filter_2": { + "type": "Biquad", + "parameters": {"type": "Highpass", "freq": 2000.0}, + "description": "Filter 2: ON HP Fc 2000 Hz", + }, + }, + "mixers": { + "Copy_1": { + "channels": {"in": 4, "out": 4}, + "mapping": [ + { + "dest": 2, + "mute": False, + "sources": [ + {"channel": 0, "gain": 0, "inverted": False, "scale": "dB"} + ], + }, + { + "dest": 3, + "mute": False, + "sources": [ + {"channel": 1, "gain": 0, "inverted": False, "scale": "dB"} + ], + }, + { + "dest": 0, + "mute": False, + "sources": [ + { + "channel": 0, + "gain": 0.0, + "inverted": False, + "scale": "dB", + } + ], + }, + { + "dest": 1, + "mute": False, + "sources": [ + { + "channel": 1, + "gain": 0.0, + "inverted": False, + "scale": "dB", + } + ], + }, + ], + "description": "Copy: RL=L RR=R", + } + }, + "pipeline": [ + {"type": "Mixer", "name": "Copy_1"}, + { + "type": "Filter", + "names": ["Filter_1"], + "description": "Channel: L R", + "channel": 0, + }, + { + "type": "Filter", + "names": ["Filter_1"], + "description": "Channel: L R", + "channel": 1, + }, + { + "type": "Filter", + "names": ["Filter_2"], + "description": "Channel: RL RR", + "channel": 2, + }, + { + "type": "Filter", + "names": ["Filter_2"], + "description": "Channel: RL RR", + "channel": 3, + }, + ], +} + + +def test_crossover(): + converter = EqAPO(CROSSOVER_EQAPO, 4) + converter.translate_file() + conf = converter.build_config() + assert conf == CROSSOVER_CDSP From 03f2482202c6164bc4674f7b617a2bd46820a6de Mon Sep 17 00:00:00 2001 From: Henrik Date: Thu, 8 Feb 2024 21:56:48 +0100 Subject: [PATCH 27/37] Separate api and aiohttp log levels --- main.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 7653cc1..9d2b4aa 100644 --- a/main.py +++ b/main.py @@ -10,15 +10,23 @@ from backend.views import version_string +def parse_logging_level(level): + level_str = level.upper() + if level_str in ("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"): + return getattr(logging, level_str) + print(f"Unknown logging level {level_str}, using default WARNING") + return logging.WARNING + level = logging.WARNING +level_aiohttp = logging.WARNING + if len(sys.argv) > 1: - level_str = sys.argv[1].upper() - if level_str in ("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"): - level = getattr(logging, level_str) - else: - print(f"Unknown logging level {level_str}, using default WARNING") + level = parse_logging_level(sys.argv[1]) + +if len(sys.argv) > 2: + level_aiohttp = parse_logging_level(sys.argv[2]) -logging.getLogger("aiohttp").setLevel(logging.WARNING) +logging.getLogger("aiohttp").setLevel(level_aiohttp) logging.getLogger("root").setLevel(level) #logging.info("info") From 641b614965e3e338fa470984d8df01cacf5b986f Mon Sep 17 00:00:00 2001 From: Henrik Date: Thu, 8 Feb 2024 21:57:32 +0100 Subject: [PATCH 28/37] Use logging in eqapo converter --- backend/eqapo_config_import.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/backend/eqapo_config_import.py b/backend/eqapo_config_import.py index 072276b..5e90ac9 100644 --- a/backend/eqapo_config_import.py +++ b/backend/eqapo_config_import.py @@ -1,4 +1,5 @@ from copy import copy, deepcopy +import logging class EqAPO: @@ -93,7 +94,7 @@ def parse_single_parameter(self, params): value = float(params[2]) parsed = {"bandwidth": value} else: - print("Skipping unknown token:", params[0]) + logging.warning("Skipping unknown token:", params[0]) return {}, params[1:] return parsed, params[nbr_tokens:] @@ -104,7 +105,7 @@ def parse_filter_params(self, param_str): ftype = params[1] ftype_c = self.filter_types.get(ftype) if not ftype_c: - print(f"Unsupported filter type '{ftype}'") + logging.warning(f"Unsupported filter type '{ftype}'") return None param_dict = {"type": ftype_c} tokens = params[2:] @@ -118,7 +119,7 @@ def parse_gain(self, param_str): params = param_str.split() gain = float(params[0]) if params[1].lower() != "db": - print("invalid preamp line:", param_str) + logging.warning("invalid preamp line:", param_str) return return {"type": "Gain", "parameters": {"gain": gain, "scale": "dB"}} @@ -144,7 +145,7 @@ def parse_copy(self, param_str): dest_ch, expr = dest.split("=") dest_ch = self.lookup_channel_index(dest_ch) handled_channels.add(dest_ch) - print("dest", dest_ch) + logging.debug("dest", dest_ch) mapping = {"dest": dest_ch, "mute": False, "sources": []} mixer["mapping"].append(mapping) sources = expr.split("+") @@ -168,7 +169,7 @@ def parse_copy(self, param_str): if channel is not None: channel = self.lookup_channel_index(channel) # TODO make a mixer config - print("source", channel, gain, scale) + logging.debug("source", channel, gain, scale) source = { "channel": channel, "gain": gain, @@ -177,7 +178,7 @@ def parse_copy(self, param_str): } mapping["sources"].append(source) for dest_ch in set(range(self.nbr_channels)) - handled_channels: - print("pass through", dest_ch) + logging.debug("pass through", dest_ch) mapping = { "dest": dest_ch, "mute": False, @@ -200,7 +201,7 @@ def parse_line(self, line): filtname = None command_name, params = line.split(":", 1) command = command_name.split()[0] - print("Parse command:", command) + logging.debug("Parse command:", command) if command in ("Filter", "Convolution", "Preamp", "Delay"): if command == "Filter": filterparams = self.parse_filter_params(params) @@ -265,14 +266,14 @@ def parse_line(self, line): "Stage", "GraphicEQ", ): - print(f"Command '{command}' is not supported, skipping.") + logging.warning(f"Command '{command}' is not supported, skipping.") else: - print(f"Skipping unrecognized command '{command}'") + logging.warning(f"Skipping unrecognized command '{command}'") def postprocess(self): for idx, step in enumerate(list(self.pipeline)): if step["type"] == "Filter" and len(step["names"]) == 0: - print("remove", step) + logging.debug("remove", step) self.pipeline.remove(step) for _, mixer in self.mixers.items(): for idx, dest in enumerate(list(mixer["mapping"])): From a869525866d5e5edbd469bbd4b4891b9fe814d2f Mon Sep 17 00:00:00 2001 From: Henrik Date: Thu, 8 Feb 2024 21:58:39 +0100 Subject: [PATCH 29/37] Add endpoint for eqapo import, add cache control headers to reduce browser disk writes --- backend/routes.py | 2 ++ backend/views.py | 74 +++++++++++++++++++++++++---------------- tests/test_basic_api.py | 25 ++++++++++++++ 3 files changed, 73 insertions(+), 28 deletions(-) diff --git a/backend/routes.py b/backend/routes.py index 9e1228f..2cd1be4 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -13,6 +13,7 @@ config_to_yml, yaml_to_json, translate_convolver_to_json, + translate_eqapo_to_json, parse_and_validate_yml_config_to_json, validate_config, get_gui_index, @@ -52,6 +53,7 @@ def setup_routes(app): app.router.add_post("/api/ymlconfigtojsonconfig", parse_and_validate_yml_config_to_json) app.router.add_post("/api/ymltojson", yaml_to_json) app.router.add_post("/api/convolvertojson", translate_convolver_to_json) + app.router.add_post("/api/eqapotojson", translate_eqapo_to_json) app.router.add_post("/api/validateconfig", validate_config) app.router.add_get("/api/storedconfigs", get_stored_configs) app.router.add_get("/api/storedcoeffs", get_stored_coeffs) diff --git a/backend/views.py b/backend/views.py index a0cc21b..3c9e701 100644 --- a/backend/views.py +++ b/backend/views.py @@ -18,6 +18,7 @@ from .filters import defaults_for_filter, filter_plot_options, pipeline_step_plot_options from .settings import get_gui_config_or_defaults from .convolver_config_import import ConvolverConfig +from .eqapo_config_import import EqAPO from .legacy_config_import import migrate_legacy_config OFFLINE_CACHE = { @@ -33,6 +34,7 @@ "clippedsamples": None, "processingload": None } +HEADERS = {"Cache-Control": "no-store"} async def get_gui_index(request): """ @@ -99,7 +101,7 @@ async def get_status(request): reconnect_thread = threading.Thread(target=_reconnect, args=(cdsp, cache), daemon=True) reconnect_thread.start() request.app["STORE"]["reconnect_thread"] = reconnect_thread - return web.json_response(cache) + return web.json_response(cache, headers=HEADERS) def version_string(version_array): @@ -135,7 +137,7 @@ async def get_param(request): result = cdsp.status.processing_load() else: raise web.HTTPNotFound(text=f"Unknown parameter {name}") - return web.Response(text=str(result)) + return web.Response(text=str(result), headers=HEADERS) async def get_list_param(request): @@ -150,7 +152,7 @@ async def get_list_param(request): result = cdsp.levels.playback_peak() else: result = "[]" - return web.json_response(result) + return web.json_response(result, headers=HEADERS) async def set_param(request): @@ -175,7 +177,7 @@ async def set_param(request): cdsp.config.set_file_path(value) elif name == "configraw": cdsp.config.set_active_raw(value) - return web.Response(text="OK") + return web.Response(text="OK", headers=HEADERS) async def eval_filter_values(request): @@ -204,7 +206,7 @@ async def eval_filter_values(request): ) data["channels"] = channels data["options"] = options - return web.json_response(data) + return web.json_response(data, headers=HEADERS) except FileNotFoundError: raise web.HTTPNotFound(text="Filter coefficient file not found") except Exception as e: @@ -236,7 +238,7 @@ async def eval_filterstep_values(request): ) data["channels"] = channels data["options"] = options - return web.json_response(data) + return web.json_response(data, headers=HEADERS) except FileNotFoundError: raise web.HTTPNotFound(text="Filter coefficient file not found") except Exception as e: @@ -248,7 +250,7 @@ async def get_config(request): """ cdsp = request.app["CAMILLA"] config = cdsp.config.active() - return web.json_response(config) + return web.json_response(config, headers=HEADERS) async def set_config(request): @@ -270,7 +272,7 @@ async def set_config(request): validator.validate_config(config_object_with_absolute_filter_paths) errors = validator.get_errors() if len(errors) > 0: - return web.json_response(data=errors) + return web.json_response(data=errors, headers=HEADERS) return web.Response(text="OK") @@ -293,7 +295,7 @@ async def get_default_config_file(request): logging.error("Failed to get default config file") traceback.print_exc() raise web.HTTPInternalServerError(text=str(e)) - return web.json_response(config_object) + return web.json_response(config_object, headers=HEADERS) async def get_active_config_file(request): """ @@ -322,7 +324,7 @@ async def get_active_config_file(request): data = {"configFileName": active_config_path, "config": config_object} else: data = {"config": config_object} - return web.json_response(data) + return web.json_response(data, headers=HEADERS) async def set_active_config_name(request): @@ -333,7 +335,7 @@ async def set_active_config_name(request): config_name = json["name"] config_file = path_of_configfile(request, config_name) set_path_as_active_config(request, config_file) - return web.Response(text="OK") + return web.Response(text="OK", headers=HEADERS) async def get_config_file(request): @@ -347,7 +349,7 @@ async def get_config_file(request): config_object = make_config_filter_paths_relative(read_yaml_from_path_to_object(request, config_file), config_dir) except CamillaError as e: raise web.HTTPInternalServerError(text=str(e)) - return web.json_response(config_object) + return web.json_response(config_object, headers=HEADERS) async def save_config_file(request): @@ -356,7 +358,7 @@ async def save_config_file(request): """ content = await request.json() save_config_to_yaml_file(content["filename"], content["config"], request) - return web.Response(text="OK") + return web.Response(text="OK", headers=HEADERS) async def config_to_yml(request): @@ -365,7 +367,7 @@ async def config_to_yml(request): """ content = await request.json() conf_yml = yaml.dump(content) - return web.Response(text=conf_yml) + return web.Response(text=conf_yml, headers=HEADERS) async def parse_and_validate_yml_config_to_json(request): @@ -376,7 +378,7 @@ async def parse_and_validate_yml_config_to_json(request): validator = request.app["VALIDATOR"] validator.validate_yamlstring(config_yaml) config = validator.get_config() - return web.json_response(config) + return web.json_response(config, headers=HEADERS) async def yaml_to_json(request): @@ -388,7 +390,7 @@ async def yaml_to_json(request): config_yaml = await request.text() loaded = yaml.safe_load(config_yaml) migrate_legacy_config(loaded) - return web.json_response(loaded) + return web.json_response(loaded, headers=HEADERS) async def translate_convolver_to_json(request): @@ -398,9 +400,25 @@ async def translate_convolver_to_json(request): """ config = await request.text() translated = ConvolverConfig(config).to_object() - return web.json_response(translated) + return web.json_response(translated, headers=HEADERS) +async def translate_eqapo_to_json(request): + """ + Parse a Convolver config string and return + as a CamillaDSP config serialized as json. + """ + try: + channels = int(request.rel_url.query.get('channels', None)) + except (ValueError, TypeError) as e: + raise web.HTTPBadRequest(reason=str(e)) + print(channels) + config = await request.text() + converter = EqAPO(config, channels) + converter.translate_file() + translated = converter.build_config() + return web.json_response(translated, headers=HEADERS) + async def validate_config(request): """ Validate a config, returned a list of errors or OK. @@ -415,7 +433,7 @@ async def validate_config(request): if len(errors) > 0: logging.debug(errors) return web.json_response(status=406, data=errors) - return web.Response(text="OK") + return web.Response(text="OK", headers=HEADERS) async def store_coeffs(request): @@ -440,7 +458,7 @@ async def get_stored_coeffs(request): """ coeff_dir = request.app["coeff_dir"] coeffs = list_of_files_in_directory(coeff_dir) - return web.json_response(coeffs) + return web.json_response(coeffs, headers=HEADERS) async def get_stored_configs(request): @@ -449,7 +467,7 @@ async def get_stored_configs(request): """ config_dir = request.app["config_dir"] configs = list_of_files_in_directory(config_dir) - return web.json_response(configs) + return web.json_response(configs, headers=HEADERS) async def delete_coeffs(request): @@ -459,7 +477,7 @@ async def delete_coeffs(request): coeff_dir = request.app["coeff_dir"] files = await request.json() delete_files(coeff_dir, files) - return web.Response(text="ok") + return web.Response(text="ok", headers=HEADERS) async def delete_configs(request): @@ -469,7 +487,7 @@ async def delete_configs(request): config_dir = request.app["config_dir"] files = await request.json() delete_files(config_dir, files) - return web.Response(text="ok") + return web.Response(text="ok", headers=HEADERS) async def download_coeffs_zip(request): @@ -502,7 +520,7 @@ async def get_gui_config(request): gui_config["supported_playback_types"] = request.app["supported_playback_types"] gui_config["can_update_active_config"] = request.app["can_update_active_config"] logging.debug(f"GUI config: {str(gui_config)}") - return web.json_response(gui_config) + return web.json_response(gui_config, headers=HEADERS) async def get_defaults_for_coeffs(request): @@ -512,7 +530,7 @@ async def get_defaults_for_coeffs(request): path = request.query["file"] absolute_path = make_absolute(path, request.app["config_dir"]) defaults = defaults_for_filter(absolute_path) - return web.json_response(defaults) + return web.json_response(defaults, headers=HEADERS) async def get_log_file(request): @@ -523,7 +541,7 @@ async def get_log_file(request): try: with open(expanduser(log_file_path)) as log_file: text = log_file.read() - return web.Response(body=text) + return web.Response(body=text, headers=HEADERS) except OSError: logging.error("Unable to read logfile at " + log_file_path) if log_file_path: @@ -540,7 +558,7 @@ async def get_capture_devices(request): backend = request.match_info["backend"] cdsp = request.app["CAMILLA"] devs = cdsp.general.list_capture_devices(backend) - return web.json_response(devs) + return web.json_response(devs, headers=HEADERS) async def get_playback_devices(request): @@ -550,7 +568,7 @@ async def get_playback_devices(request): backend = request.match_info["backend"] cdsp = request.app["CAMILLA"] devs = cdsp.general.list_playback_devices(backend) - return web.json_response(devs) + return web.json_response(devs, headers=HEADERS) async def get_backends(request): @@ -559,4 +577,4 @@ async def get_backends(request): """ cdsp = request.app["CAMILLA"] backends = cdsp.general.supported_device_types() - return web.json_response(backends) + return web.json_response(backends, headers=HEADERS) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 0398dd9..97cc8e1 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -249,3 +249,28 @@ async def test_active_config_offline(offline_server): print(content) assert content["configFileName"] == "config2.yml" assert content["config"]["devices"]["samplerate"] == 48000 + + +@pytest.mark.asyncio +async def test_translate_eqapo(server): + from test_eqapo_config_import import EXAMPLE + + resp = await server.post("/api/eqapotojson?channels=2", data=EXAMPLE) + assert resp.status == 200 + content = await resp.json() + assert "filters" in content + + +@pytest.mark.asyncio +async def test_translate_eqapo_bad(server): + resp = await server.post("/api/eqapotojson", data="blank") + assert resp.status == 400 + + +@pytest.mark.asyncio +async def test_translate_convolver(server): + resp = await server.post("/api/convolvertojson", data="96000 1 2 0\n0\n0") + assert resp.status == 200 + content = await resp.json() + assert "devices" in content + assert content["devices"]["samplerate"] == 96000 From fe3f737ba6e0772a5940acd8bec53eadc220b55b Mon Sep 17 00:00:00 2001 From: Henrik Date: Mon, 12 Feb 2024 17:39:48 +0100 Subject: [PATCH 30/37] Better parsing of numbers in eqapo config --- backend/eqapo_config_import.py | 35 +++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/backend/eqapo_config_import.py b/backend/eqapo_config_import.py index 5e90ac9..5f224b8 100644 --- a/backend/eqapo_config_import.py +++ b/backend/eqapo_config_import.py @@ -33,7 +33,7 @@ class EqAPO: 2: {"L": 0, "R": 1}, 4: {"L": 0, "R": 1, "RL": 2, "RR": 3}, 6: {"L": 0, "R": 1, "C": 2, "LFE": 3, "RL": 4, "RR": 5}, - 8: {"L": 0, "R": 1, "C": 2, "LFE": 3, "RL": 4, "RR": 5, "SL": 6, "SR": 7} + 8: {"L": 0, "R": 1, "C": 2, "LFE": 3, "RL": 4, "RR": 5, "SL": 6, "SR": 7}, } delay_units = {"ms": "ms", "samples": "samples"} @@ -58,7 +58,9 @@ def __init__(self, config_text, nbr_channels): } self.pipeline = [self.current_filterstep] self.nbr_channels = nbr_channels - self.channel_map = self.all_channel_maps.get(nbr_channels, self.all_channel_maps[8]) + self.channel_map = self.all_channel_maps.get( + nbr_channels, self.all_channel_maps[8] + ) def lookup_channel_index(self, label): if label in self.channel_map: @@ -66,10 +68,21 @@ def lookup_channel_index(self, label): elif label.isnumeric(): channel = int(label) - 1 else: - # TODO handle this + logging.warning( + f"Virtual channels are not supported, skipping channel {label}" + ) channel = None return channel + def parse_number(self, value_str): + try: + return float(value_str) + except ValueError: + logging.warning( + f"Unable to parse '{value_str}' as number, inline expressions are not supported." + ) + return None + # Parse a single command parameter def parse_single_parameter(self, params): # Inline expressions (ex: Fc `2*a`) are not supported @@ -77,21 +90,21 @@ def parse_single_parameter(self, params): if params[0] == "Fc": nbr_tokens = 3 assert params[2].lower() == "hz" - value = float(params[1]) + value = self.parse_number(params[1]) parsed = {"freq": value} elif params[0] == "Q": nbr_tokens = 2 - value = float(params[1]) + value = self.parse_number(params[1]) parsed = {"q": value} elif params[0] == "Gain": nbr_tokens = 3 assert params[2].lower() == "db" - value = float(params[1]) + value = self.parse_number(params[1]) parsed = {"gain": value} elif params[0] == "BW": nbr_tokens = 3 assert params[1].lower() == "oct" - value = float(params[2]) + value = self.parse_number(params[2]) parsed = {"bandwidth": value} else: logging.warning("Skipping unknown token:", params[0]) @@ -117,7 +130,7 @@ def parse_filter_params(self, param_str): # Parse a Preamp command to a filter def parse_gain(self, param_str): params = param_str.split() - gain = float(params[0]) + gain = self.parse_number(params[0]) if params[1].lower() != "db": logging.warning("invalid preamp line:", param_str) return @@ -126,7 +139,7 @@ def parse_gain(self, param_str): # Parse a Delay command to a filter def parse_delay(self, param_str): params = param_str.split() - delay = float(params[0]) + delay = self.parse_number(params[0]) unit = self.delay_units[params[1]] return {"type": "Delay", "parameters": {"delay": delay, "unit": unit}} @@ -153,10 +166,10 @@ def parse_copy(self, param_str): if "*" in source: gain_str, channel = source.split("*") if gain_str.endswith("dB"): - gain = float(gain_str[:-2]) + gain = self.parse_number(gain_str[:-2]) scale = "dB" else: - gain = float(gain_str) + gain = self.parse_number(gain_str) scale = "linear" elif source == "0.0": # EqAPO supports setting channels to an arbitrary constant. From b3cf458003f76c11ef5c366a0d7d142524904b9c Mon Sep 17 00:00:00 2001 From: Henrik Date: Mon, 12 Feb 2024 18:03:36 +0100 Subject: [PATCH 31/37] Add missing headers --- backend/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/views.py b/backend/views.py index 3c9e701..84c18b1 100644 --- a/backend/views.py +++ b/backend/views.py @@ -273,7 +273,7 @@ async def set_config(request): errors = validator.get_errors() if len(errors) > 0: return web.json_response(data=errors, headers=HEADERS) - return web.Response(text="OK") + return web.Response(text="OK", headers=HEADERS) async def get_default_config_file(request): @@ -548,7 +548,7 @@ async def get_log_file(request): error_message = "Please configure CamillaDSP to log to: " + log_file_path else: error_message = "Please configure a valid 'log_file' path" - return web.Response(body=error_message) + return web.Response(body=error_message, headers=HEADERS) async def get_capture_devices(request): From 15590c86eba6d71b30a63c37af88a8f3e4c66168 Mon Sep 17 00:00:00 2001 From: Henrik Date: Thu, 15 Feb 2024 23:17:56 +0100 Subject: [PATCH 32/37] Add support for certificates to enable https --- README.md | 61 ++++++++++++++++++++++++++++++++----------- config/camillagui.yml | 2 ++ main.py | 8 +++++- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 8c85da3..ae20238 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ Unzip the file and edit `config/camillagui.yml` as needed, see [Configuration](# The Python dependencies are listed in three different files, for use with different Python package/environment managers. - `cdsp_conda.yml` for [conda](https://conda.io/). -- `requirements.txt` for [pip](https://pip.pypa.io/), often combined with an environment manager such as [venv](https://docs.python.org/3/library/venv.html). +- `requirements.txt` for [pip](https://pip.pypa.io/), often combined with an environment manager + such as [venv](https://docs.python.org/3/library/venv.html). - `pyproject.toml` for [Poetry](https://python-poetry.org). @@ -42,6 +43,8 @@ camilla_host: "0.0.0.0" camilla_port: 1234 bind_address: "0.0.0.0" port: 5005 +ssl_certificate: null (*) +ssl_private_key: null (*) config_dir: "~/camilladsp/configs" coeff_dir: "~/camilladsp/coeffs" default_config: "~/camilladsp/default_config.yml" @@ -52,11 +55,33 @@ on_get_active_config: null (*) supported_capture_types: null (*) supported_playback_types: null (*) ``` -The options marked `(*)` are optional. If left out the default values listed above will be used. The included configuration has CamillaDSP running on the same machine as the backend, with the websocket server enabled at port 1234. The web interface will be served on port 5005. It is possible to run the gui and CamillaDSP on different machines, just point the `camilla_host` to the right address. - -**Warning**: By default the backend will bind to all network interfaces. This makes the gui available on all networks the system is connected to, which may be insecure. Make sure to change the `bind_address` if you want it to be reachable only on specific network interface(s) and/or to set your firewall to block external (internet) access to this backend. +The options marked `(*)` are optional. If left out the default values listed above will be used. +The included configuration has CamillaDSP running on the same machine as the backend, +with the websocket server enabled at port 1234. +The web interface will be served on port 5005 using plain HTTP. +It is possible to run the gui and CamillaDSP on different machines, +just point the `camilla_host` to the right address. + +**Warning**: By default the backend will bind to all network interfaces. +This makes the gui available on all networks the system is connected to, which may be insecure. +Make sure to change the `bind_address` if you want it to be reachable only on specific +network interface(s) and/or to set your firewall to block external (internet) access to this backend. + +The `ssl_certificate` and `ssl_private_key` options are used to configure SSL, to enable HTTPS. +Both a certificate and a private key are required. +The values for `ssl_certificate` and `ssl_private_key` should then be the paths to the files containing the certificate and key. +It's also possible to keep both certificate and key in a single file. +In that case, provide only `ssl_certificate`. +See the [Python ssl documentation](https://docs.python.org/3/library/ssl.html#ssl-certificates) +for more info on certificates. + +To generate a self-signed certificate and key pair, use openssl: +```sh +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout my_private_key.key -out my_certificate.crt +``` -The settings for config_dir and coeff_dir point to two folders where the backend has permissions to write files. This is provided to enable uploading of coefficients and config files from the gui. +The settings for config_dir and coeff_dir point to two folders where the backend has permissions to write files. +This is provided to enable uploading of coefficients and config files from the gui. If you want to be able to view the log file in the GUI, configure CamillaDSP to log to `log_file`. @@ -81,7 +106,8 @@ See also [Integrating with other software](#integrating-with-other-software) ### Limit device types -By default, the config validator allows all the device types that CamillaDSP can support. To limit this to the types that are supported on a particular system, give the list of supported types as: +By default, the config validator allows all the device types that CamillaDSP can support. +To limit this to the types that are supported on a particular system, give the list of supported types as: ```yaml supported_capture_types: ["Alsa", "File", "Stdin"] supported_playback_types: ["Alsa", "File", "Stdout"] @@ -89,7 +115,7 @@ supported_playback_types: ["Alsa", "File", "Stdout"] ### Adding custom shortcut settings It is possible to configure custom shortcuts for the `Shortcuts` section and the compact view. -Here is an example config to set the gain of a filter called `MyFilter` within the range from 0 to 10 db in steps of 0.1 dB. +Here is an example config to set the gain of a filter called `MyFilter` within the range from 0 to 10 db in steps of 0.1 dB. ``` custom_shortcuts: - section: "My custom section" @@ -112,7 +138,8 @@ there are some options to customize the UI for your particular needs. #### Setting and getting the active config _NOTE: This functionality is experimental, there may be significant changes in future versions._ -The configuration options `on_set_active_config` and `on_get_active_config` can be used to customize the way the active config file path is stored. +The configuration options `on_set_active_config` and `on_get_active_config` can be used to customize +the way the active config file path is stored. These are shell commands that will be run to set and get the active config. Setting these options will override the normal way of getting and setting the active config path. Since the commands are run in the operating system shell, the syntax depends on which operating system is used. @@ -123,9 +150,9 @@ This means it must contain an empty set of curly brackets, where the filename wi Examples: - Running a script: `on_set_active_config: my_updater_script.sh {}` - + The backend will run the command: `my_updater_script.sh "/full/path/to/new_active_config.yml"` -- Saving config filename to a text file: `on_set_active_config: echo {} > active_configname.txt` +- Saving config filename to a text file: `on_set_active_config: echo {} > active_configname.txt` The backend will run the command: `echo "/full/path/to/new_active_config.yml" > active_configname.txt` @@ -152,7 +179,7 @@ hide_rate_monitoring: false Changes to the currently edited config can be applied automatically, but this behavior is disabled by default. To enable it by default, in `config/gui-config.yml` set `apply_config_automatically` to `true`. -The update rate of the level meters can be adjusted by changing the `status_update_interval` setting. +The update rate of the level meters can be adjusted by changing the `status_update_interval` setting. The value is in milliseconds, and the default value is 100 ms. ## Running @@ -163,19 +190,23 @@ python main.py The gui should now be available at: http://localhost:5005/gui/index.html -If accessing the gui from a different machine, replace "localhost" by the IP or hostname of the machine running the gui server. +If accessing the gui from a different machine, replace "localhost" by the IP +or hostname of the machine running the gui server. ## Development ### Render the environment files -This repository contains [jinja](https://palletsprojects.com/p/jinja/) templates used to create the Python environment files. +This repository contains [jinja](https://palletsprojects.com/p/jinja/) +templates used to create the Python environment files. The templates are stored in `release_automation/templates/`. -To render the templates, install the dependencies `PyYAML` and `jinja2` and run the Python script `render_env_files.py`: +To render the templates, install the dependencies `PyYAML` and `jinja2` +and run the Python script `render_env_files.py`: ```sh python -m release_automation.render_env_files ``` -When rendering, the versions of the Python dependencies are taken from the file `release_automation/versions.yml`. +When rendering, the versions of the Python dependencies are taken +from the file `release_automation/versions.yml`. The backend version is read from `backend/version.py`. ### Running the tests diff --git a/config/camillagui.yml b/config/camillagui.yml index d2b9e07..a6108e1 100644 --- a/config/camillagui.yml +++ b/config/camillagui.yml @@ -3,6 +3,8 @@ camilla_host: "127.0.0.1" camilla_port: 1234 bind_address: "0.0.0.0" port: 5005 +ssl_certificate: null +ssl_private_key: null config_dir: "~/camilladsp/configs" coeff_dir: "~/camilladsp/coeffs" default_config: "~/camilladsp/default_config.yml" diff --git a/main.py b/main.py index 9d2b4aa..8abd370 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ from aiohttp import web +import ssl import logging import sys import camilladsp @@ -68,7 +69,12 @@ def build_app(backend_config): def main(): app = build_app(config) - web.run_app(app, host=config["bind_address"], port=config["port"]) + if config.get("ssl_certificate"): + ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + ssl_context.load_cert_chain(config["ssl_certificate"], keyfile=config.get("ssl_private_key")) + else: + ssl_context = None + web.run_app(app, host=config["bind_address"], port=config["port"], ssl_context=ssl_context) if __name__ == "__main__": main() From 88981481f0a8545a6d71402bbcf2d72047a1d81f Mon Sep 17 00:00:00 2001 From: Henrik Date: Fri, 16 Feb 2024 21:07:49 +0100 Subject: [PATCH 33/37] Store backends and device lists, update on reconnect --- backend/views.py | 40 ++++++++++++++++++++++++++++++++++------ main.py | 14 +++++++++----- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/backend/views.py b/backend/views.py index 84c18b1..e99f051 100644 --- a/backend/views.py +++ b/backend/views.py @@ -42,12 +42,28 @@ async def get_gui_index(request): """ raise web.HTTPFound("/gui/index.html") -def _reconnect(cdsp, cache): +def _reconnect(cdsp, cache, validator): done = False while not done: try: cdsp.connect() cache["cdsp_version"] = version_string(cdsp.versions.camilladsp()) + # Update backends + backends = cdsp.general.supported_device_types() + cache["backends"] = backends + pb_backends, cap_backends = backends + logging.debug(f"Updated backends: {backends}") + validator.set_supported_capture_types(cap_backends) + validator.set_supported_playback_types(pb_backends) + # Update playback and capture devices + for pb_backend in pb_backends: + pb_devs = cdsp.general.list_playback_devices(pb_backend) + logging.debug(f"Updated {pb_backend} playback devices: {pb_devs}") + cache["playback_devices"][pb_backend] = pb_devs + for cap_backend in cap_backends: + cap_devs = cdsp.general.list_capture_devices(cap_backend) + logging.debug(f"Updated {cap_backend} capture devices: {cap_devs}") + cache["capture_devices"][cap_backend] = cap_devs done = True except IOError: time.sleep(1) @@ -62,6 +78,7 @@ async def get_status(request): reconnect_thread = request.app["STORE"]["reconnect_thread"] cache = request.app["STATUSCACHE"] cachetime = request.app["STORE"]["cache_time"] + validator = request.app["VALIDATOR"] try: levels_since = float(request.query.get("since")) except: @@ -98,7 +115,7 @@ async def get_status(request): except IOError: if reconnect_thread is None or not reconnect_thread.is_alive(): cache.update(OFFLINE_CACHE) - reconnect_thread = threading.Thread(target=_reconnect, args=(cdsp, cache), daemon=True) + reconnect_thread = threading.Thread(target=_reconnect, args=(cdsp, cache, validator), daemon=True) reconnect_thread.start() request.app["STORE"]["reconnect_thread"] = reconnect_thread return web.json_response(cache, headers=HEADERS) @@ -554,27 +571,38 @@ async def get_log_file(request): async def get_capture_devices(request): """ Get a list of available capture devices for a backend. + Return a cached list if CamillaDSP is offline. """ backend = request.match_info["backend"] cdsp = request.app["CAMILLA"] - devs = cdsp.general.list_capture_devices(backend) + try: + devs = cdsp.general.list_capture_devices(backend) + except IOError: + logging.debug("CamillaDSP is offline, returning capture devices from cache") + devs = request.app["STATUSCACHE"]["capture_devices"].get(backend, []) return web.json_response(devs, headers=HEADERS) async def get_playback_devices(request): """ Get a list of available playback devices for a backend. + Return a cached list if CamillaDSP is offline. """ backend = request.match_info["backend"] cdsp = request.app["CAMILLA"] - devs = cdsp.general.list_playback_devices(backend) + try: + devs = cdsp.general.list_playback_devices(backend) + except IOError: + logging.debug("CamillaDSP is offline, returning playback devices from cache") + devs = request.app["STATUSCACHE"]["playback_devices"].get(backend, []) return web.json_response(devs, headers=HEADERS) async def get_backends(request): """ Get lists of available playback and capture backends. + Since this can not change while CamillaDSP is running, + the response is taken from the cache. """ - cdsp = request.app["CAMILLA"] - backends = cdsp.general.supported_device_types() + backends = request.app["STATUSCACHE"]["backends"] return web.json_response(backends, headers=HEADERS) diff --git a/main.py b/main.py index 8abd370..0032778 100644 --- a/main.py +++ b/main.py @@ -53,11 +53,15 @@ def build_app(backend_config): app["CAMILLA"] = camilladsp.CamillaClient(backend_config["camilla_host"], backend_config["camilla_port"]) app["STATUSCACHE"] = { "backend_version": version_string(VERSION), - "py_cdsp_version": version_string(app["CAMILLA"].versions.library()) - } - app["STORE"] = {} - app["STORE"]["reconnect_thread"] = None - app["STORE"]["cache_time"] = 0 + "py_cdsp_version": version_string(app["CAMILLA"].versions.library()), + "backends": [], + "playback_devices": {}, + "capture_devices": {}, + } + app["STORE"] = { + "reconnect_thread": None, + "cache_time": 0, + } camillavalidator = CamillaValidator() if backend_config["supported_capture_types"] is not None: From b8efe1b402f17dfe4b64b373870282122f0a544d Mon Sep 17 00:00:00 2001 From: Henrik Date: Sat, 24 Feb 2024 22:49:20 +0100 Subject: [PATCH 34/37] Bump versions --- backend/version.py | 2 +- release_automation/versions.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/version.py b/backend/version.py index b89310f..08ad377 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1 +1 @@ -VERSION = (2, 0, 1) \ No newline at end of file +VERSION = (2, 1, 0) \ No newline at end of file diff --git a/release_automation/versions.yml b/release_automation/versions.yml index 025585e..8583c36 100644 --- a/release_automation/versions.yml +++ b/release_automation/versions.yml @@ -1,4 +1,4 @@ --- -camillagui_tag: v2.0.0 +camillagui_tag: v2.1.0 pycamilladsp_tag: v2.0.2 pycamilladsp_plot_tag: v2.0.0 From 924333517c188249a5425d2cc0c8c7f58122f135 Mon Sep 17 00:00:00 2001 From: Henrik Date: Sat, 24 Feb 2024 23:17:22 +0100 Subject: [PATCH 35/37] Make compatible with python 3.8, format code --- backend/convolver_config_import.py | 42 ++++++------ backend/filemanagement.py | 73 +++++++++++++------- backend/filters.py | 56 ++++++++++------ backend/routes.py | 6 +- backend/settings.py | 20 +++--- backend/version.py | 2 +- backend/views.py | 97 +++++++++++++++++++-------- tests/test_convolver_config_import.py | 2 +- 8 files changed, 192 insertions(+), 106 deletions(-) diff --git a/backend/convolver_config_import.py b/backend/convolver_config_import.py index 4f53d18..d077996 100644 --- a/backend/convolver_config_import.py +++ b/backend/convolver_config_import.py @@ -1,4 +1,5 @@ from os.path import basename +from typing import List, Tuple def filename_of_path(path: str) -> str: @@ -18,7 +19,7 @@ def fraction_to_gain(fraction: str) -> float: return float(f"0.{fraction}") -def parse_channel_and_fraction(channel: str, fraction: str) -> (int, float, bool): +def parse_channel_and_fraction(channel: str, fraction: str) -> Tuple[int, float, bool]: int_channel = abs(int(channel)) gain = fraction_to_gain(fraction) inverted = channel.startswith("-") @@ -27,7 +28,7 @@ def parse_channel_and_fraction(channel: str, fraction: str) -> (int, float, bool def channels_factors_and_inversions_as_list( channels_and_factors: str, -) -> list[tuple[int, float, bool]]: +) -> List[Tuple[int, float, bool]]: channels_and_fractions = [ channel_and_fraction.split(".") for channel_and_fraction in channels_and_factors.split(" ") @@ -38,7 +39,7 @@ def channels_factors_and_inversions_as_list( ] -def make_filter_step(channel: int, names: list[str]) -> dict: +def make_filter_step(channel: int, names: List[str]) -> dict: return { "type": "Filter", "channel": channel, @@ -48,7 +49,7 @@ def make_filter_step(channel: int, names: list[str]) -> dict: } -def make_mixer_mapping(input_channels: list[tuple], output_channel: int) -> dict: +def make_mixer_mapping(input_channels: List[tuple], output_channel: int) -> dict: return { "dest": output_channel, "sources": [ @@ -67,10 +68,10 @@ class Filter: filename: str channel: int channel_in_file: int - input_channels: list[tuple[int, float, bool]] - output_channels: list[tuple[int, float, bool]] + input_channels: List[Tuple[int, float, bool]] + output_channels: List[Tuple[int, float, bool]] - def __init__(self, channel, filter_text: list[str]): + def __init__(self, channel, filter_text: List[str]): self.channel = channel self.filename = filename_of_path(filter_text[0]) self.channel_in_file = int(filter_text[1]) @@ -85,9 +86,9 @@ class ConvolverConfig: _samplerate: int _input_channels: int _output_channels: int - _input_delays: list[int] - _output_delays: list[int] - _filters: list[Filter] + _input_delays: List[int] + _output_delays: List[int] + _filters: List[Filter] def __init__(self, config_text: str): """ @@ -107,11 +108,14 @@ def __init__(self, config_text: str): ] def to_object(self) -> dict: + filters = self._delay_filter_definitions() + filters.update(self._convolution_filter_definitions()) + mixers = self._mixer_in() + mixers.update(self._mixer_out()) return { "devices": {"samplerate": self._samplerate}, - "filters": self._delay_filter_definitions() - | self._convolution_filter_definitions(), - "mixers": self._mixer_in() | self._mixer_out(), + "filters": filters, + "mixers": mixers, "pipeline": self._input_delay_pipeline_steps() + self._mixer_in_pipeline_step() + self._filter_pipeline_steps() @@ -148,17 +152,17 @@ def _convolution_filter_definitions(self) -> dict: for f in self._filters } - def _input_delay_pipeline_steps(self) -> list[dict]: + def _input_delay_pipeline_steps(self) -> List[dict]: return self._delay_pipeline_steps(self._input_delays) - def _delay_pipeline_steps(self, delays: list[int]) -> list[dict]: + def _delay_pipeline_steps(self, delays: List[int]) -> List[dict]: return [ make_filter_step(channel, [self._delay_name(delay)]) for channel, delay in enumerate(delays) if delay != 0 ] - def _output_delay_pipeline_steps(self) -> list[dict]: + def _output_delay_pipeline_steps(self) -> List[dict]: return self._delay_pipeline_steps(self._output_delays) def _mixer_in(self) -> dict: @@ -198,12 +202,12 @@ def _mixer_out(self) -> dict: } @staticmethod - def _mixer_in_pipeline_step() -> list[dict]: + def _mixer_in_pipeline_step() -> List[dict]: return [{"type": "Mixer", "name": "Mixer in", "description": None}] @staticmethod - def _mixer_out_pipeline_step() -> list[dict]: + def _mixer_out_pipeline_step() -> List[dict]: return [{"type": "Mixer", "name": "Mixer out", "description": None}] - def _filter_pipeline_steps(self) -> list[dict]: + def _filter_pipeline_steps(self) -> List[dict]: return [make_filter_step(f.channel, [f.name()]) for f in self._filters] diff --git a/backend/filemanagement.py b/backend/filemanagement.py index 2534433..3f5a80b 100644 --- a/backend/filemanagement.py +++ b/backend/filemanagement.py @@ -2,7 +2,17 @@ import os import zipfile from copy import deepcopy -from os.path import isfile, split, join, realpath, relpath, normpath, isabs, commonpath, getmtime +from os.path import ( + isfile, + split, + join, + realpath, + relpath, + normpath, + isabs, + commonpath, + getmtime, +) import logging import traceback @@ -18,11 +28,12 @@ "volume": [0.0, 0.0, 0.0, 0.0, 0.0], } + def file_in_folder(folder, filename): """ Safely join a folder and filename. """ - if '/' in filename or '\\' in filename: + if "/" in filename or "\\" in filename: raise IOError("Filename may not contain any slashes/backslashes") return os.path.abspath(os.path.join(folder, filename)) @@ -55,26 +66,24 @@ def list_of_files_in_directory(folder): """ Return a list of files (name and modification date) in a folder. """ - files = [file_in_folder(folder, file) - for file in os.listdir(folder) - if isfile(file_in_folder(folder, file))] + files = [ + file_in_folder(folder, file) + for file in os.listdir(folder) + if isfile(file_in_folder(folder, file)) + ] files_list = map( lambda file: { "name": (os.path.basename(file)), - "lastModified": (getmtime(file)) + "lastModified": (getmtime(file)), }, - files + files, ) sorted_files = sorted(files_list, key=lambda x: x["name"].lower()) return sorted_files def list_of_filenames_in_directory(folder): - return map( - lambda file: file["name"], - list_of_files_in_directory(folder) - ) - + return map(lambda file: file["name"], list_of_files_in_directory(folder)) def delete_files(folder, files): @@ -106,7 +115,7 @@ def zip_of_files(folder, files): with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) as zip_file: for file_name in files: file_path = file_in_folder(folder, file_name) - with open(file_path, 'r') as file: + with open(file_path, "r") as file: zip_file.write(file_path, file_name) return zip_buffer.getvalue() @@ -142,13 +151,17 @@ def get_active_config_path(request): logging.debug(filename) return filename else: - logging.error("CamillaDSP runs without state file and is unable to persistently store config file path") + logging.error( + "CamillaDSP runs without state file and is unable to persistently store config file path" + ) return None elif statefile_path: confpath = _read_statefile_config_path(statefile_path) return _verify_path_in_config_dir(confpath, config_dir) else: - logging.error("The backend config has no state file and is unable to persistently store config file path") + logging.error( + "The backend config has no state file and is unable to persistently store config file path" + ) else: try: logging.debug(f"Running command: {on_get}") @@ -184,14 +197,18 @@ def set_path_as_active_config(request, filepath): logging.error(f"Failed to update statefile at {statefile_path}") traceback.print_exc() else: - logging.error("The backend config has no state file and is unable to persistently store config file path") + logging.error( + "The backend config has no state file and is unable to persistently store config file path" + ) else: dsp_statefile_path = cdsp.general.state_file_path() if dsp_statefile_path: logging.debug(f"Send set config file path command with '{filepath}'") cdsp.config.set_file_path(filepath) else: - logging.error("CamillaDSP runs without state file and is unable to persistently store config file path") + logging.error( + "CamillaDSP runs without state file and is unable to persistently store config file path" + ) if on_set: try: cmd = on_set.format(f'"{filepath}"') @@ -201,6 +218,7 @@ def set_path_as_active_config(request, filepath): logging.error(f"Failed to run on_set_active_config command") traceback.print_exc() + def _verify_path_in_config_dir(path, config_dir): """ Verify that a given path points to a file in config_dir. @@ -213,7 +231,9 @@ def _verify_path_in_config_dir(path, config_dir): if is_path_in_folder(canonical, config_dir): _head, tail = split(canonical) return tail - logging.error(f"The config file path '{path}' is not in the config dir '{config_dir}'") + logging.error( + f"The config file path '{path}' is not in the config dir '{config_dir}'" + ) return None @@ -233,7 +253,7 @@ def _update_statefile_config_path(statefile_path, new_config_path): logging.error(f"Details: {e}") state = deepcopy(DEFAULT_STATEFILE) state["config_path"] = new_config_path - yaml_state = yaml.dump(state).encode('utf-8') + yaml_state = yaml.dump(state).encode("utf-8") with open(statefile_path, "wb") as f: f.write(yaml_state) @@ -254,12 +274,13 @@ def _read_statefile_config_path(statefile_path): logging.error(f"Details: {e}") return None + def save_config_to_yaml_file(config_name, config_object, request): """ Write a given config object to a yaml file. """ config_file = path_of_configfile(request, config_name) - yaml_config = yaml.dump(config_object).encode('utf-8') + yaml_config = yaml.dump(config_object).encode("utf-8") with open(config_file, "wb") as f: f.write(yaml_config) @@ -268,8 +289,10 @@ def coeff_dir_relative_to_config_dir(request): """ Get the relative path of the coeff_dir with respect to config_dir. """ - relative_coeff_dir = relpath(request.app["coeff_dir"], start=request.app["config_dir"]) - coeff_dir_with_folder_separator_at_end = join(relative_coeff_dir, '') + relative_coeff_dir = relpath( + request.app["coeff_dir"], start=request.app["config_dir"] + ) + coeff_dir_with_folder_separator_at_end = join(relative_coeff_dir, "") return coeff_dir_with_folder_separator_at_end @@ -337,9 +360,11 @@ def replace_tokens_in_filter_config(filterconfig, samplerate, channels): ftype = filterconfig["type"] parameters = filterconfig["parameters"] if ftype == "Conv" and parameters["type"] in ["Raw", "Wav"]: - parameters["filename"] = parameters["filename"]\ - .replace("$samplerate$", str(samplerate))\ + parameters["filename"] = ( + parameters["filename"] + .replace("$samplerate$", str(samplerate)) .replace("$channels$", str(channels)) + ) def make_relative(path, base_dir): diff --git a/backend/filters.py b/backend/filters.py index 7c22cd4..4e4fa93 100644 --- a/backend/filters.py +++ b/backend/filters.py @@ -2,16 +2,16 @@ from os.path import splitext, basename FORMAT_MAP = { - ".txt": "TEXT", - ".csv": "TEXT", + ".txt": "TEXT", + ".csv": "TEXT", ".tsv": "TEXT", ".dbl": "FLOAT64LE", ".raw": "S32LE", ".pcm": "S32LE", ".dat": "S32LE", ".sam": "S32LE", - ".f32": "FLOAT32LE", - ".f64": "FLOAT64LE", + ".f32": "FLOAT32LE", + ".f64": "FLOAT64LE", ".i32": "S32LE", ".i24": "S24LE3", ".i16": "S16LE", @@ -60,8 +60,9 @@ def pattern_from_filter_file_name(path): Regex patterns for matching samplerate and channels tokens in filename. """ filename = re.escape(basename(path)) - pattern = filename.replace(r"\$samplerate\$", "(?P\\d*)")\ - .replace(r"\$channels\$", "(?P\\d*)") + pattern = filename.replace(r"\$samplerate\$", "(?P\\d*)").replace( + r"\$channels\$", "(?P\\d*)" + ) return re.compile(pattern) @@ -70,10 +71,16 @@ def pipeline_step_plot_options(filter_file_names, config, step_index): Get the combined available samplerate and channels options for a filter step. """ samplerates_and_channels_for_filter = map_of_samplerates_and_channels_per_filter( - config, filter_file_names,step_index) - all_samplerate_and_channel_options = set_of_all_samplerate_and_channel_options(samplerates_and_channels_for_filter) - samplerate_and_channel_options = set_of_samplerate_and_channel_options_available_for_all_filters( - all_samplerate_and_channel_options, samplerates_and_channels_for_filter) + config, filter_file_names, step_index + ) + all_samplerate_and_channel_options = set_of_all_samplerate_and_channel_options( + samplerates_and_channels_for_filter + ) + samplerate_and_channel_options = ( + set_of_samplerate_and_channel_options_available_for_all_filters( + all_samplerate_and_channel_options, samplerates_and_channels_for_filter + ) + ) return plot_options_to_object(samplerate_and_channel_options) @@ -90,21 +97,27 @@ def map_of_samplerates_and_channels_per_filter(config, filter_file_names, step_i parameters = filter["parameters"] if filter["type"] == "Conv" and parameters["type"] in {"Raw", "Wav"}: filename = parameters["filename"] - samplerates_and_channels_for_filter[filter_name] = samplerate_and_channel_pairs_from_options( + samplerates_and_channels_for_filter[ + filter_name + ] = samplerate_and_channel_pairs_from_options( filter_plot_options(filter_file_names, filename), default_samplerate, - default_channels + default_channels, ) return samplerates_and_channels_for_filter -def samplerate_and_channel_pairs_from_options(options, default_samplerate, default_channels): +def samplerate_and_channel_pairs_from_options( + options, default_samplerate, default_channels +): """ Make a set of unique (samplerate, channels) pairs from a list of options. """ pairs = set() for option in options: - samplerate = option["samplerate"] if "samplerate" in option else default_samplerate + samplerate = ( + option["samplerate"] if "samplerate" in option else default_samplerate + ) channels = option["channels"] if "channels" in option else default_channels pairs.add((samplerate, channels)) return pairs @@ -116,18 +129,23 @@ def set_of_all_samplerate_and_channel_options(samplerates_and_channels_for_filte """ samplerate_and_channel_options = set() for filter_name in samplerates_and_channels_for_filter: - samplerate_and_channel_options.update(samplerates_and_channels_for_filter[filter_name]) + samplerate_and_channel_options.update( + samplerates_and_channels_for_filter[filter_name] + ) return samplerate_and_channel_options -def set_of_samplerate_and_channel_options_available_for_all_filters(samplerate_and_channel_options, - samplerates_and_channels_for_filter): +def set_of_samplerate_and_channel_options_available_for_all_filters( + samplerate_and_channel_options, samplerates_and_channels_for_filter +): """ Append additional values to an existing set. """ options_available_for_all_filters = set(samplerate_and_channel_options) for filter_name in samplerates_and_channels_for_filter: - options_available_for_all_filters.intersection_update(samplerates_and_channels_for_filter[filter_name]) + options_available_for_all_filters.intersection_update( + samplerates_and_channels_for_filter[filter_name] + ) return options_available_for_all_filters @@ -143,7 +161,7 @@ def plot_options_to_object(samplerate_and_channel_options): { "name": str(samplerate) + " Hz - " + str(channels) + " Channels", "samplerate": samplerate, - "channels": channels + "channels": channels, } ) step_options.sort(key=lambda x: x["name"]) diff --git a/backend/routes.py b/backend/routes.py index 2cd1be4..d5c60e9 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -33,7 +33,7 @@ get_log_file, get_capture_devices, get_playback_devices, - get_backends + get_backends, ) @@ -50,7 +50,9 @@ def setup_routes(app): app.router.add_get("/api/getdefaultconfigfile", get_default_config_file) app.router.add_post("/api/setactiveconfigfile", set_active_config_name) app.router.add_post("/api/configtoyml", config_to_yml) - app.router.add_post("/api/ymlconfigtojsonconfig", parse_and_validate_yml_config_to_json) + app.router.add_post( + "/api/ymlconfigtojsonconfig", parse_and_validate_yml_config_to_json + ) app.router.add_post("/api/ymltojson", yaml_to_json) app.router.add_post("/api/convolvertojson", translate_convolver_to_json) app.router.add_post("/api/eqapotojson", translate_eqapo_to_json) diff --git a/backend/settings.py b/backend/settings.py index c9c9c23..b163777 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -8,8 +8,8 @@ import logging BASEPATH = pathlib.Path(__file__).parent.parent.absolute() -CONFIG_PATH = BASEPATH / 'config' / 'camillagui.yml' -GUI_CONFIG_PATH = BASEPATH / 'config' / 'gui-config.yml' +CONFIG_PATH = BASEPATH / "config" / "camillagui.yml" +GUI_CONFIG_PATH = BASEPATH / "config" / "gui-config.yml" # Default values for the optional gui config. GUI_CONFIG_DEFAULTS = { @@ -59,14 +59,10 @@ def get_config(path): config = _load_yaml(path) if config is None: sys.exit() - config["config_dir"] = os.path.abspath( - os.path.expanduser(config["config_dir"])) - config["coeff_dir"] = os.path.abspath( - os.path.expanduser(config["coeff_dir"])) - config["default_config"] = absolute_path_or_none_if_empty( - config["default_config"]) - config["statefile_path"] = absolute_path_or_none_if_empty( - config["statefile_path"]) + config["config_dir"] = os.path.abspath(os.path.expanduser(config["config_dir"])) + config["coeff_dir"] = os.path.abspath(os.path.expanduser(config["coeff_dir"])) + config["default_config"] = absolute_path_or_none_if_empty(config["default_config"]) + config["statefile_path"] = absolute_path_or_none_if_empty(config["statefile_path"]) for key, value in BACKEND_CONFIG_DEFAULTS.items(): if key not in config: config[key] = value @@ -90,7 +86,9 @@ def can_update_active_config(config): else: logging.error(f"The statefile {statefile} is not writable.") if config["on_set_active_config"] and config["on_get_active_config"]: - logging.debug("Both 'on_set_active_config' and 'on_get_active_config' options are set") + logging.debug( + "Both 'on_set_active_config' and 'on_get_active_config' options are set" + ) external_supported = True return statefile_supported or external_supported diff --git a/backend/version.py b/backend/version.py index 08ad377..96177d1 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1 +1 @@ -VERSION = (2, 1, 0) \ No newline at end of file +VERSION = (2, 1, 0) diff --git a/backend/views.py b/backend/views.py index e99f051..39735c9 100644 --- a/backend/views.py +++ b/backend/views.py @@ -9,13 +9,29 @@ import traceback from .filemanagement import ( - path_of_configfile, store_files, list_of_files_in_directory, delete_files, - zip_response, zip_of_files, read_yaml_from_path_to_object, set_path_as_active_config, get_active_config_path, save_config_to_yaml_file, - make_config_filter_paths_absolute, coeff_dir_relative_to_config_dir, - replace_relative_filter_path_with_absolute_paths, make_config_filter_paths_relative, - make_absolute, replace_tokens_in_filter_config, list_of_filenames_in_directory + path_of_configfile, + store_files, + list_of_files_in_directory, + delete_files, + zip_response, + zip_of_files, + read_yaml_from_path_to_object, + set_path_as_active_config, + get_active_config_path, + save_config_to_yaml_file, + make_config_filter_paths_absolute, + coeff_dir_relative_to_config_dir, + replace_relative_filter_path_with_absolute_paths, + make_config_filter_paths_relative, + make_absolute, + replace_tokens_in_filter_config, + list_of_filenames_in_directory, +) +from .filters import ( + defaults_for_filter, + filter_plot_options, + pipeline_step_plot_options, ) -from .filters import defaults_for_filter, filter_plot_options, pipeline_step_plot_options from .settings import get_gui_config_or_defaults from .convolver_config_import import ConvolverConfig from .eqapo_config_import import EqAPO @@ -32,16 +48,18 @@ "rateadjust": None, "bufferlevel": None, "clippedsamples": None, - "processingload": None + "processingload": None, } HEADERS = {"Cache-Control": "no-store"} + async def get_gui_index(request): """ Serve the static gui files. """ raise web.HTTPFound("/gui/index.html") + def _reconnect(cdsp, cache, validator): done = False while not done: @@ -68,6 +86,7 @@ def _reconnect(cdsp, cache, validator): except IOError: time.sleep(1) + async def get_status(request): """ Get the state and singnal levels etc. @@ -92,30 +111,36 @@ async def get_status(request): levels = cdsp.levels.levels_since(levels_since) else: levels = cdsp.levels.levels() - cache.update({ - "capturesignalrms": levels["capture_rms"], - "capturesignalpeak": levels["capture_peak"], - "playbacksignalrms": levels["playback_rms"], - "playbacksignalpeak": levels["playback_peak"], - }) + cache.update( + { + "capturesignalrms": levels["capture_rms"], + "capturesignalpeak": levels["capture_peak"], + "playbacksignalrms": levels["playback_rms"], + "playbacksignalpeak": levels["playback_peak"], + } + ) now = time.time() # These values don't change that fast, let's update them only once per second. if now - cachetime > 1.0: request.app["STORE"]["cache_time"] = now - cache.update({ - "capturerate": cdsp.rate.capture(), - "rateadjust": cdsp.status.rate_adjust(), - "bufferlevel": cdsp.status.buffer_level(), - "clippedsamples": cdsp.status.clipped_samples(), - "processingload": cdsp.status.processing_load() - }) + cache.update( + { + "capturerate": cdsp.rate.capture(), + "rateadjust": cdsp.status.rate_adjust(), + "bufferlevel": cdsp.status.buffer_level(), + "clippedsamples": cdsp.status.clipped_samples(), + "processingload": cdsp.status.processing_load(), + } + ) except IOError as e: print("TODO safe to remove this try-except? error:", e) pass except IOError: if reconnect_thread is None or not reconnect_thread.is_alive(): cache.update(OFFLINE_CACHE) - reconnect_thread = threading.Thread(target=_reconnect, args=(cdsp, cache, validator), daemon=True) + reconnect_thread = threading.Thread( + target=_reconnect, args=(cdsp, cache, validator), daemon=True + ) reconnect_thread.start() request.app["STORE"]["reconnect_thread"] = reconnect_thread return web.json_response(cache, headers=HEADERS) @@ -229,6 +254,7 @@ async def eval_filter_values(request): except Exception as e: raise web.HTTPBadRequest(text=str(e)) + async def eval_filterstep_values(request): """ Evaluate a filter step consisting of one or several filters. Returns values for plotting. @@ -261,6 +287,7 @@ async def eval_filterstep_values(request): except Exception as e: raise web.HTTPBadRequest(text=str(e)) + async def get_config(request): """ Get running config. @@ -279,13 +306,15 @@ async def set_config(request): config_dir = request.app["config_dir"] cdsp = request.app["CAMILLA"] validator = request.app["VALIDATOR"] - config_object_with_absolute_filter_paths = make_config_filter_paths_absolute(config_object, config_dir) + config_object_with_absolute_filter_paths = make_config_filter_paths_absolute( + config_object, config_dir + ) if cdsp.is_connected(): try: cdsp.config.set_active(config_object_with_absolute_filter_paths) except CamillaError as e: raise web.HTTPInternalServerError(text=str(e)) - else: + else: validator.validate_config(config_object_with_absolute_filter_paths) errors = validator.get_errors() if len(errors) > 0: @@ -304,7 +333,9 @@ async def get_default_config_file(request): else: raise web.HTTPNotFound(text="No default config") try: - config_object = make_config_filter_paths_relative(read_yaml_from_path_to_object(request, config), config_dir) + config_object = make_config_filter_paths_relative( + read_yaml_from_path_to_object(request, config), config_dir + ) except CamillaError as e: logging.error(f"Failed to get default config file, error: {e}") raise web.HTTPInternalServerError(text=str(e)) @@ -314,6 +345,7 @@ async def get_default_config_file(request): raise web.HTTPInternalServerError(text=str(e)) return web.json_response(config_object, headers=HEADERS) + async def get_active_config_file(request): """ Get the active config. If no config is active, return the default config. @@ -329,7 +361,9 @@ async def get_active_config_file(request): else: raise web.HTTPNotFound(text="No active or default config") try: - config_object = make_config_filter_paths_relative(read_yaml_from_path_to_object(request, config), config_dir) + config_object = make_config_filter_paths_relative( + read_yaml_from_path_to_object(request, config), config_dir + ) except CamillaError as e: logging.error(f"Failed to get active config from CamillaDSP, error: {e}") raise web.HTTPInternalServerError(text=str(e)) @@ -363,7 +397,9 @@ async def get_config_file(request): config_name = request.query["name"] config_file = path_of_configfile(request, config_name) try: - config_object = make_config_filter_paths_relative(read_yaml_from_path_to_object(request, config_file), config_dir) + config_object = make_config_filter_paths_relative( + read_yaml_from_path_to_object(request, config_file), config_dir + ) except CamillaError as e: raise web.HTTPInternalServerError(text=str(e)) return web.json_response(config_object, headers=HEADERS) @@ -426,7 +462,7 @@ async def translate_eqapo_to_json(request): as a CamillaDSP config serialized as json. """ try: - channels = int(request.rel_url.query.get('channels', None)) + channels = int(request.rel_url.query.get("channels", None)) except (ValueError, TypeError) as e: raise web.HTTPBadRequest(reason=str(e)) print(channels) @@ -436,16 +472,19 @@ async def translate_eqapo_to_json(request): translated = converter.build_config() return web.json_response(translated, headers=HEADERS) + async def validate_config(request): """ Validate a config, returned a list of errors or OK. """ config_dir = request.app["config_dir"] config = await request.json() - config_with_absolute_filter_paths = make_config_filter_paths_absolute(config, config_dir) + config_with_absolute_filter_paths = make_config_filter_paths_absolute( + config, config_dir + ) validator = request.app["VALIDATOR"] validator.validate_config(config_with_absolute_filter_paths) - #print(yaml.dump(config_with_absolute_filter_paths, indent=2)) + # print(yaml.dump(config_with_absolute_filter_paths, indent=2)) errors = validator.get_errors() if len(errors) > 0: logging.debug(errors) diff --git a/tests/test_convolver_config_import.py b/tests/test_convolver_config_import.py index e2b8a0a..b9e15b8 100644 --- a/tests/test_convolver_config_import.py +++ b/tests/test_convolver_config_import.py @@ -11,7 +11,7 @@ def clean_multi_line_string(multiline_text: str): :param multiline_text: :return: the text without the first blank line and indentation """ - return dedent(multiline_text.removeprefix("\n")) + return dedent(multiline_text.lstrip("\n")) def test_filename_of_path(): From 6269ce57e04b9cd4dc98886734c854ecbda0975a Mon Sep 17 00:00:00 2001 From: Henrik Date: Sun, 25 Feb 2024 10:30:37 +0100 Subject: [PATCH 36/37] More logical structure in readme --- README.md | 59 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ae20238..77b64c1 100644 --- a/README.md +++ b/README.md @@ -113,24 +113,6 @@ supported_capture_types: ["Alsa", "File", "Stdin"] supported_playback_types: ["Alsa", "File", "Stdout"] ``` -### Adding custom shortcut settings -It is possible to configure custom shortcuts for the `Shortcuts` section and the compact view. -Here is an example config to set the gain of a filter called `MyFilter` within the range from 0 to 10 db in steps of 0.1 dB. -``` -custom_shortcuts: - - section: "My custom section" - description: "Optional description for the section. Omit the attribute, if unwanted" - shortcuts: - - name: "My filter gain" - description: "Optional description for the setting. Omit the attribute, if unwanted" - path_in_config: ["filters", "MyFilter", "parameters", "gain"] - range_from: 0 - range_to: 10 - step: 0.1 - - name: "The next setting" - ... -``` - ### Integrating with other software If you want to integrate CamillaGUI with other software, there are some options to customize the UI for your particular needs. @@ -159,12 +141,37 @@ Examples: The `on_get_active_config` command is expected to return a filename on stdout. As an example, read a filename from a text file: `on_get_active_config: "cat myconfig.txt"`. -#### Styling the GUI -The UI can be styled by editing `build/css-variables.css`. -Further instructions on how to do this, or switch back to the brighter black/white UI, can be found there. -#### Hiding GUI Options -Options can hidden from your users by editing `config/gui-config.yml`. + +## Customizing the GUI +Some functionality of the GUI can be customized by editing `config/gui-config.yml`. +The styling can be customized by editing `build/css-variables.css`. + +### Adding custom shortcut settings +It is possible to configure custom shortcuts for the `Shortcuts` section and the compact view. +The included config file contains the default Bass and Treble filters. +To add more, edit the file `config/gui-config.yml` to add +the new shortcuts to the list under `custom_shortcuts`. + +Here is an example config to set the gain of a filter called `MyFilter` +within the range from 0 to 10 db in steps of 0.1 dB. +```yaml +custom_shortcuts: + - section: "My custom section" + description: "Optional description for the section. Omit the attribute, if unwanted" + shortcuts: + - name: "My filter gain" + description: "Optional description for the setting. Omit the attribute, if unwanted" + path_in_config: ["filters", "MyFilter", "parameters", "gain"] + range_from: 0 + range_to: 10 + step: 0.1 + - name: "The next setting" + ... +``` + +### Hiding GUI Options +Options can be hidden from your users by editing `config/gui-config.yml`. Setting any of the options to `true` hides the corresponding option or section. These are all optional, and default to `false` if left out. ```yaml @@ -175,7 +182,11 @@ hide_playback_device: false hide_rate_monitoring: false ``` -#### Other GUI Options +### Styling the GUI +The UI can be styled by editing `build/css-variables.css`. +Further instructions on how to do this, or switch back to the brighter black/white UI, can be found there. + +### Other GUI Options Changes to the currently edited config can be applied automatically, but this behavior is disabled by default. To enable it by default, in `config/gui-config.yml` set `apply_config_automatically` to `true`. From 9e54496f80cc1cbc4a336261f8cb0a4116c2f3c9 Mon Sep 17 00:00:00 2001 From: Henrik Date: Mon, 26 Feb 2024 09:09:10 +0100 Subject: [PATCH 37/37] Add config file validation, list all dependencies --- README.md | 6 + backend/settings.py | 28 ++++- backend/settings_schemas.py | 106 ++++++++++++++++++ .../templates/cdsp_conda.yml.j2 | 2 + .../templates/pyproject.toml.j2 | 1 + .../templates/requirements.txt.j2 | 2 + 6 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 backend/settings_schemas.py diff --git a/README.md b/README.md index 77b64c1..f3de415 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,12 @@ custom_shortcuts: ... ``` +The gui config is checked when the backend starts, and any problems are logged. +For example, `range_from` must be a number. If it is not, this results in a message such as this: +``` +ERROR:root:Parameter 'custom_shortcuts/0/shortcuts/1/range_from': 'hello' is not of type 'number' +``` + ### Hiding GUI Options Options can be hidden from your users by editing `config/gui-config.yml`. Setting any of the options to `true` hides the corresponding option or section. diff --git a/backend/settings.py b/backend/settings.py index b163777..a19edca 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -4,9 +4,12 @@ import yaml from yaml.scanner import ScannerError +from jsonschema import Draft202012Validator import logging +from .settings_schemas import GUI_CONFIG_SCHEMA, BACKEND_CONFIG_SCHEMA + BASEPATH = pathlib.Path(__file__).parent.parent.absolute() CONFIG_PATH = BASEPATH / "config" / "camillagui.yml" GUI_CONFIG_PATH = BASEPATH / "config" / "gui-config.yml" @@ -24,6 +27,7 @@ # Default values for the optional settings. BACKEND_CONFIG_DEFAULTS = { + "default_config": None, "statefile_path": None, "on_set_active_config": None, "on_get_active_config": None, @@ -51,12 +55,25 @@ def _load_yaml(path): return None +def _read_and_validate_file(path, schema): + config = _load_yaml(path) + if config is None: + return None + validator = Draft202012Validator(schema) + errors = list(validator.iter_errors(config)) + if len(errors) > 0: + logging.error(f"Error in config file '{path}'") + for e in errors: + logging.error(f"Parameter '{'/'.join([str(p) for p in e.path])}': {e.message}") + return None + return config + def get_config(path): """ Get backend config. Exits if the config can't be read. """ - config = _load_yaml(path) + config = _read_and_validate_file(path, BACKEND_CONFIG_SCHEMA) if config is None: sys.exit() config["config_dir"] = os.path.abspath(os.path.expanduser(config["config_dir"])) @@ -68,7 +85,14 @@ def get_config(path): config[key] = value logging.debug("Backend configuration:") logging.debug(yaml.dump(config)) + config["can_update_active_config"] = can_update_active_config(config) + + # Read the gui config. + # This is only to validate the file and log any problems. + # The result is not used. + get_gui_config_or_defaults() + return config @@ -132,7 +156,7 @@ def get_gui_config_or_defaults(): Get the gui config from file if it exists, if not return the defaults. """ - config = _load_yaml(GUI_CONFIG_PATH) + config = _read_and_validate_file(GUI_CONFIG_PATH, GUI_CONFIG_SCHEMA) if config is not None: for key, value in GUI_CONFIG_DEFAULTS.items(): if key not in config: diff --git a/backend/settings_schemas.py b/backend/settings_schemas.py new file mode 100644 index 0000000..f2c714c --- /dev/null +++ b/backend/settings_schemas.py @@ -0,0 +1,106 @@ +BACKEND_CONFIG_SCHEMA = { + "type": "object", + "properties": { + "camilla_host": {"type": "string", "minLength": 1}, + "camilla_port": { + "type": "integer", + }, + "bind_address": {"type": "string", "minLength": 1}, + "port": { + "type": "integer", + }, + "ssl_certificate": {"type": ["string", "null"], "minLength": 1}, + "ssl_private_key": {"type": ["string", "null"], "minLength": 1}, + "config_dir": {"type": "string", "minLength": 1}, + "coeff_dir": {"type": "string", "minLength": 1}, + "default_config": {"type": ["string", "null"], "minLength": 1}, + "statefile_path": {"type": ["string", "null"], "minLength": 1}, + "log_file": {"type": ["string", "null"], "minLength": 1}, + "on_set_active_config": {"type": ["string", "null"], "minLength": 1}, + "on_get_active_config": {"type": ["string", "null"], "minLength": 1}, + "supported_capture_types": { + "type": ["array", "null"], + "items": {"type": "string", "minLength": 1}, + }, + "supported_playback_types": { + "type": ["array", "null"], + "items": {"type": "string", "minLength": 1}, + }, + }, + "required": [ + "camilla_host", + "camilla_port", + "bind_address", + "port", + "config_dir", + "coeff_dir", + ], +} + +GUI_CONFIG_SCHEMA = { + "type": "object", + "properties": { + "hide_capture_samplerate": {"type": "boolean"}, + "hide_silence": {"type": "boolean"}, + "hide_capture_device": {"type": "boolean"}, + "hide_playback_device": {"type": "boolean"}, + "apply_config_automatically": {"type": "boolean"}, + "save_config_automatically": {"type": "boolean"}, + "status_update_interval": {"type": "integer", "minValue": 1}, + "custom_shortcuts": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "section": {"type": "string"}, + "description": {"type": "string"}, + "shortcuts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "path_in_config": { + "type": "array", + "items": {"type": "string"}, + }, + "range_from": {"type": "number"}, + "range_to": {"type": "number"}, + "step": {"type": "number"}, + }, + "required": [ + "name", + "path_in_config", + "range_from", + "range_to", + "step", + ], + }, + }, + }, + "required": ["section", "shortcuts"], + }, + }, + }, + "required": [], +} + +""" + +custom_shortcuts: + - section: "Equalizer" + description: "To use the EQ, add filters named \"Bass\" and \"Treble\" to the pipeline.
Recommented settings:
Bass: Biquad Lowshelf freq=85 q=0.9
Treble: Biquad Highshelf freq=6500 q=0.7" + shortcuts: + - name: "Treble (dB)" + path_in_config: ["filters", "Treble", "parameters", "gain"] + range_from: -12 + range_to: 12 + step: 0.5 + - name: "Bass (dB)" + path_in_config: ["filters", "Bass", "parameters", "gain"] + range_from: -12 + range_to: 12 + step: 0.5 + + + """ diff --git a/release_automation/templates/cdsp_conda.yml.j2 b/release_automation/templates/cdsp_conda.yml.j2 index 9bae107..95a2b12 100644 --- a/release_automation/templates/cdsp_conda.yml.j2 +++ b/release_automation/templates/cdsp_conda.yml.j2 @@ -6,6 +6,8 @@ channels: dependencies: - pip - aiohttp + - jsonschema + - pyyaml - pip: - "git+https://github.com/HEnquist/pycamilladsp.git@{{ pycamilladsp_tag }}" - "camilladsp-plot[plot]@git+https://github.com/HEnquist/pycamilladsp-plot.git@{{ pycamilladsp_plot_tag }}" diff --git a/release_automation/templates/pyproject.toml.j2 b/release_automation/templates/pyproject.toml.j2 index 035705e..f2e91d9 100644 --- a/release_automation/templates/pyproject.toml.j2 +++ b/release_automation/templates/pyproject.toml.j2 @@ -12,6 +12,7 @@ websocket-client = "^1.6.4" jsonschema = "^4.20.0" aiohttp = "^3.9.0" numpy = "^1.26.0" +PyYAML = "^6.0.0" camilladsp = {git = "https://github.com/HEnquist/pycamilladsp.git", rev = "{{ pycamilladsp_tag }}"} camilladsp-plot = {git = "https://github.com/HEnquist/pycamilladsp-plot.git", rev = "{{ pycamilladsp_plot_tag }}"} diff --git a/release_automation/templates/requirements.txt.j2 b/release_automation/templates/requirements.txt.j2 index ea6f21c..bf288fe 100644 --- a/release_automation/templates/requirements.txt.j2 +++ b/release_automation/templates/requirements.txt.j2 @@ -1,4 +1,6 @@ {# Template for pip requirements file -#} aiohttp +jsonschema +PyYAML git+https://github.com/HEnquist/pycamilladsp.git@{{ pycamilladsp_tag }} camilladsp-plot[plot]@git+https://github.com/HEnquist/pycamilladsp-plot.git@{{ pycamilladsp_plot_tag }}