From edcf4c2ab6122b9c9d25009fba85025997f72397 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 5 May 2025 15:10:55 +0200 Subject: [PATCH 01/29] feat: more flexible helper for batch reduction --- src/ess/amor/load.py | 2 +- src/ess/amor/types.py | 4 - src/ess/reflectometry/tools.py | 117 +++++++++++++++++++++--------- src/ess/reflectometry/types.py | 4 + src/ess/reflectometry/workflow.py | 2 +- tests/tools_test.py | 57 +++++++++++++-- 6 files changed, 141 insertions(+), 45 deletions(-) diff --git a/src/ess/amor/load.py b/src/ess/amor/load.py index ea103497..47cb65aa 100644 --- a/src/ess/amor/load.py +++ b/src/ess/amor/load.py @@ -10,6 +10,7 @@ LoadedNeXusDetector, NeXusDetectorName, ProtonCurrent, + RawChopper, RawDetectorData, RunType, SampleRotation, @@ -21,7 +22,6 @@ ChopperFrequency, ChopperPhase, ChopperSeparation, - RawChopper, ) diff --git a/src/ess/amor/types.py b/src/ess/amor/types.py index 92eb704e..bb801398 100644 --- a/src/ess/amor/types.py +++ b/src/ess/amor/types.py @@ -27,8 +27,4 @@ class ChopperSeparation(sciline.Scope[RunType, sc.Variable], sc.Variable): """Distance between the two choppers.""" -class RawChopper(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): - """Chopper data loaded from nexus file.""" - - GravityToggle = NewType("GravityToggle", bool) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 6302824e..ee82396c 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -8,10 +8,15 @@ import sciline import scipp as sc import scipy.optimize as opt -from orsopy.fileio.orso import OrsoDataset from ess.reflectometry import orso -from ess.reflectometry.types import ReflectivityOverQ +from ess.reflectometry.types import ( + Filename, + ReducibleData, + ReflectivityOverQ, + SampleRun, +) +from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) @@ -280,18 +285,18 @@ def combine_curves( ) -def orso_datasets_from_measurements( +def from_measurements( workflow: sciline.Pipeline, - runs: Sequence[Mapping[type, Any]], + runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], + target: type | Sequence[type] = orso.OrsoIofQDataset, *, - scale_to_overlap: bool = True, -) -> list[OrsoDataset]: - '''Produces a list of ORSO datasets containing one - reflectivity curve for each of the provided runs. + scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, +) -> list | Mapping: + '''Produces a list of datasets containing for each of the provided runs. Each entry of :code:`runs` is a mapping of parameters and values needed to produce the dataset. - Optionally, the reflectivity curves can be scaled to overlap in + Optionally, the results can be scaled so that the reflectivity curves overlap in the regions where they have the same Q-value. Parameters @@ -300,38 +305,82 @@ def orso_datasets_from_measurements( The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. runs: - The sciline parameters to be used for each run + The sciline parameters to be used for each run. + + target: + The domain type to compute for each run. scale_to_overlap: - If True the curves will be scaled to overlap. - Note that the curve of the first run is unscaled and - the rest are scaled to match it. + If not ``None`` the curves will be scaled to overlap. + If a tuple then this argument will be passed as the ``critical_edge_interval`` + to the ``scale_reflectivity_curves_to_overlap`` function. Returns --------- list of the computed ORSO datasets, containing one reflectivity curve each ''' - reflectivity_curves = [] - for parameters in runs: - wf = workflow.copy() - for name, value in parameters.items(): - wf[name] = value - reflectivity_curves.append(wf.compute(ReflectivityOverQ)) - - scale_factors = ( - scale_reflectivity_curves_to_overlap([r.hist() for r in reflectivity_curves])[1] - if scale_to_overlap - else (1,) * len(runs) - ) + names = runs.keys() if hasattr(runs, 'keys') else None + runs = runs.values() if hasattr(runs, 'values') else runs + + def init_workflow(workflow, parameters): + if Filename[SampleRun] in parameters: + if isinstance(parameters[Filename[SampleRun]], list | tuple): + wf = with_filenames( + workflow, + SampleRun, + parameters[Filename[SampleRun]], + ) + else: + wf = workflow.copy() + wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] + else: + wf = workflow.copy() + for tp, value in parameters.items(): + if tp is Filename[SampleRun]: + continue + wf[tp] = value + return wf + + if scale_to_overlap: + results = [] + for parameters in runs: + wf = init_workflow(workflow, parameters) + results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) + + scale_factors = scale_reflectivity_curves_to_overlap( + [r[ReflectivityOverQ].hist() for r in results], + critical_edge_interval=scale_to_overlap + if isinstance(scale_to_overlap, list | tuple) + else None, + )[1] datasets = [] - for parameters, curve, scale_factor in zip( - runs, reflectivity_curves, scale_factors, strict=True - ): - wf = workflow.copy() - for name, value in parameters.items(): - wf[name] = value - wf[ReflectivityOverQ] = scale_factor * curve - dataset = wf.compute(orso.OrsoIofQDataset) + for i, parameters in enumerate(runs): + wf = init_workflow(workflow, parameters) + if scale_to_overlap: + # Optimization in case we can avoid re-doing some work: + # Check if any of the targets need ReducibleData if + # ReflectivityOverQ already exists. + # If they don't, we can avoid recomputing ReducibleData. + targets = target if hasattr(target, '__len__') else (target,) + if any( + _workflow_needs_quantity_A_even_if_quantitiy_B_is_set( + wf[t], ReducibleData[SampleRun], ReflectivityOverQ + ) + for t in targets + ): + wf[ReducibleData[SampleRun]] = ( + scale_factors[i] * results[i][ReducibleData[SampleRun]] + ) + else: + wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] + + dataset = wf.compute(target) datasets.append(dataset) - return datasets + return datasets if names is None else dict(zip(names, datasets, strict=True)) + + +def _workflow_needs_quantity_A_even_if_quantitiy_B_is_set(workflow, A, B): + wf = workflow.copy() + wf[B] = 'Not important' + return A in wf.underlying_graph diff --git a/src/ess/reflectometry/types.py b/src/ess/reflectometry/types.py index 4451fe48..6f02b70e 100644 --- a/src/ess/reflectometry/types.py +++ b/src/ess/reflectometry/types.py @@ -36,6 +36,10 @@ class LoadedNeXusDetector(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): """NXdetector loaded from file""" +class RawChopper(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): + """Chopper data loaded from nexus file.""" + + class ReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): """Event data with common coordinates added""" diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 2c0fa9be..4f40570f 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -6,7 +6,6 @@ import sciline import scipp as sc -from ess.amor.types import RawChopper from ess.reflectometry.orso import ( OrsoExperiment, OrsoOwner, @@ -15,6 +14,7 @@ ) from ess.reflectometry.types import ( Filename, + RawChopper, ReducibleData, RunType, SampleRotation, diff --git a/tests/tools_test.py b/tests/tools_test.py index 03c07aaf..ce71cbe9 100644 --- a/tests/tools_test.py +++ b/tests/tools_test.py @@ -8,11 +8,10 @@ from orsopy.fileio import Orso, OrsoDataset from scipp.testing import assert_allclose -from ess.reflectometry.orso import OrsoIofQDataset from ess.reflectometry.tools import ( combine_curves, + from_measurements, linlogspace, - orso_datasets_from_measurements, scale_reflectivity_curves_to_overlap, ) from ess.reflectometry.types import Filename, ReflectivityOverQ, SampleRun @@ -225,11 +224,11 @@ def test_linlogspace_bad_input(): @pytest.mark.filterwarnings("ignore:No suitable") -def test_orso_datasets_tool(): +def test_from_measurements_tool(): def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: return filename - def orso_dataset(filename: Filename[SampleRun]) -> OrsoIofQDataset: + def orso_dataset(filename: Filename[SampleRun]) -> OrsoDataset: class Reduction: corrections = [] # noqa: RUF012 @@ -240,10 +239,58 @@ class Reduction: workflow = sl.Pipeline( [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} ) - datasets = orso_datasets_from_measurements( + datasets = from_measurements( workflow, [{}, {Filename[SampleRun]: 'special'}], + target=OrsoDataset, scale_to_overlap=False, ) assert len(datasets) == 2 assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') + + +@pytest.mark.filterwarnings("ignore:No suitable") +def test_from_measurements_tool_uses_expected_parameters_from_each_run(): + def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: + return filename + + def orso_dataset(filename: Filename[SampleRun]) -> OrsoDataset: + class Reduction: + corrections = [] # noqa: RUF012 + + return OrsoDataset( + Orso({}, Reduction, [], name=f'{filename}.orso'), np.ones((0, 0)) + ) + + workflow = sl.Pipeline( + [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} + ) + datasets = from_measurements( + workflow, + [{}, {Filename[SampleRun]: 'special'}], + target=OrsoDataset, + scale_to_overlap=False, + ) + assert len(datasets) == 2 + assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') + + +@pytest.mark.parametrize('targets', [(int,), (float, int)]) +@pytest.mark.parametrize( + 'params', [[{str: '1'}, {str: '2'}], {'a': {str: '1'}, 'b': {str: '2'}}] +) +def test_from_measurements_tool_returns_mapping_if_passed_mapping(params, targets): + def A(x: str) -> float: + return float(x) + + def B(x: str) -> int: + return int(x) + + workflow = sl.Pipeline([A, B]) + datasets = from_measurements( + workflow, + params, + target=targets, + ) + assert len(datasets) == len(params) + assert type(datasets) is type(params) From 7dbbf2ce0e07972aef0c2edb85e907e3e7dadc46 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 5 May 2025 15:16:36 +0200 Subject: [PATCH 02/29] fix: remove duplicate --- tests/tools_test.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/tools_test.py b/tests/tools_test.py index ce71cbe9..40b7dc39 100644 --- a/tests/tools_test.py +++ b/tests/tools_test.py @@ -223,32 +223,6 @@ def test_linlogspace_bad_input(): ) -@pytest.mark.filterwarnings("ignore:No suitable") -def test_from_measurements_tool(): - def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: - return filename - - def orso_dataset(filename: Filename[SampleRun]) -> OrsoDataset: - class Reduction: - corrections = [] # noqa: RUF012 - - return OrsoDataset( - Orso({}, Reduction, [], name=f'{filename}.orso'), np.ones((0, 0)) - ) - - workflow = sl.Pipeline( - [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} - ) - datasets = from_measurements( - workflow, - [{}, {Filename[SampleRun]: 'special'}], - target=OrsoDataset, - scale_to_overlap=False, - ) - assert len(datasets) == 2 - assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') - - @pytest.mark.filterwarnings("ignore:No suitable") def test_from_measurements_tool_uses_expected_parameters_from_each_run(): def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: From 25add6ba22d92e401a2311465f80af2af2cc1c88 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 09:38:44 +0200 Subject: [PATCH 03/29] update docs --- docs/user-guide/amor/amor-reduction.ipynb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 3331c4de..643572eb 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -465,13 +465,15 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.reflectometry.tools import orso_datasets_from_measurements\n", + "from ess.reflectometry.tools import from_measurements\n", "\n", - "datasets = orso_datasets_from_measurements(\n", + "datasets = from_measurements(\n", " workflow,\n", " runs.values(),\n", + " target=orso.OrsoIofQDataset,\n", " # Optionally scale the curves to overlap using `scale_reflectivity_curves_to_overlap`\n", - " scale_to_overlap=True\n", + " # with the same \"critical edge interval\" that was used before\n", + " scale_to_overlap=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", ")" ] }, From eda422e85d4540fa8e8ffbf087442f571bc0284d Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 09:42:20 +0200 Subject: [PATCH 04/29] spelling --- src/ess/reflectometry/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index ee82396c..1ae5b37f 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -364,7 +364,7 @@ def init_workflow(workflow, parameters): # If they don't, we can avoid recomputing ReducibleData. targets = target if hasattr(target, '__len__') else (target,) if any( - _workflow_needs_quantity_A_even_if_quantitiy_B_is_set( + _workflow_needs_quantity_A_even_if_quantity_B_is_set( wf[t], ReducibleData[SampleRun], ReflectivityOverQ ) for t in targets @@ -380,7 +380,7 @@ def init_workflow(workflow, parameters): return datasets if names is None else dict(zip(names, datasets, strict=True)) -def _workflow_needs_quantity_A_even_if_quantitiy_B_is_set(workflow, A, B): +def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): wf = workflow.copy() wf[B] = 'Not important' return A in wf.underlying_graph From 98a938925db2c71985a45544d37c3b7878989ed8 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 13:11:22 +0200 Subject: [PATCH 05/29] fix: add theta to reference, can be useful in some contexts --- src/ess/amor/normalization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ess/amor/normalization.py b/src/ess/amor/normalization.py index eb97d188..d4c450b0 100644 --- a/src/ess/amor/normalization.py +++ b/src/ess/amor/normalization.py @@ -88,6 +88,7 @@ def evaluate_reference_at_sample_coords( ref = ref.transform_coords( ( "Q", + "theta", "wavelength_resolution", "sample_size_resolution", "angular_resolution", From 743743a7874d0355bea2fe0fe084adfa066f05ef Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 13:56:05 +0200 Subject: [PATCH 06/29] fix: handle case when SampleRotation etc are set in workflow --- src/ess/reflectometry/workflow.py | 38 +++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 4f40570f..76dfe734 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -13,6 +13,7 @@ OrsoSampleFilenames, ) from ess.reflectometry.types import ( + DetectorRotation, Filename, RawChopper, ReducibleData, @@ -62,15 +63,34 @@ def with_filenames( mapped = wf.map(df) - wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( - index=axis_name, func=_concatenate_event_lists - ) - wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( - index=axis_name, func=_any_value - ) - wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( - index=axis_name, func=_any_value - ) + try: + wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( + index=axis_name, func=_concatenate_event_lists + ) + except ValueError: + # ReducibleData[runtype] is independent of Filename[runtype] + pass + try: + wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # RawChopper[runtype] is independent of Filename[runtype] + pass + try: + wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # SampleRotation[runtype] is independent of Filename[runtype] + pass + try: + wf[DetectorRotation[runtype]] = mapped[DetectorRotation[runtype]].reduce( + index=axis_name, func=_any_value + ) + except ValueError: + # DetectorRotation[runtype] is independent of Filename[runtype] + pass if runtype is SampleRun: wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) From 45bcd0ea969bbb998ead0f1651be7dd704048308 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 14:04:30 +0200 Subject: [PATCH 07/29] fix: add parameters before setting filenames --- src/ess/reflectometry/tools.py | 17 ++++----- tests/amor/tools_test.py | 68 ++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 tests/amor/tools_test.py diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 1ae5b37f..c06d6c3b 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -308,7 +308,7 @@ def from_measurements( The sciline parameters to be used for each run. target: - The domain type to compute for each run. + The domain type(s) to compute for each run. scale_to_overlap: If not ``None`` the curves will be scaled to overlap. @@ -323,22 +323,21 @@ def from_measurements( runs = runs.values() if hasattr(runs, 'values') else runs def init_workflow(workflow, parameters): + wf = workflow.copy() + for tp, value in parameters.items(): + if tp is Filename[SampleRun]: + continue + wf[tp] = value + if Filename[SampleRun] in parameters: if isinstance(parameters[Filename[SampleRun]], list | tuple): wf = with_filenames( - workflow, + wf, SampleRun, parameters[Filename[SampleRun]], ) else: - wf = workflow.copy() wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] - else: - wf = workflow.copy() - for tp, value in parameters.items(): - if tp is Filename[SampleRun]: - continue - wf[tp] = value return wf if scale_to_overlap: diff --git a/tests/amor/tools_test.py b/tests/amor/tools_test.py new file mode 100644 index 00000000..0dd6fef2 --- /dev/null +++ b/tests/amor/tools_test.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import pytest +import sciline +import scipp as sc +from scipp.testing import assert_allclose + +from amor.pipeline_test import amor_pipeline # noqa: F401 +from ess.amor import data +from ess.amor.types import ChopperPhase +from ess.reflectometry.tools import from_measurements +from ess.reflectometry.types import ( + DetectorRotation, + Filename, + QBins, + ReducedReference, + ReferenceRun, + ReflectivityOverQ, + SampleRotation, + SampleRun, +) + +# The files used in the AMOR reduction workflow have some scippnexus warnings +pytestmark = pytest.mark.filterwarnings( + "ignore:.*Invalid transformation, .*missing attribute 'vector':UserWarning", +) + + +@pytest.fixture +def pipeline_with_1632_reference(amor_pipeline): # noqa: F811 + amor_pipeline[ChopperPhase[ReferenceRun]] = sc.scalar(7.5, unit='deg') + amor_pipeline[ChopperPhase[SampleRun]] = sc.scalar(7.5, unit='deg') + amor_pipeline[Filename[ReferenceRun]] = data.amor_run('1632') + amor_pipeline[ReducedReference] = amor_pipeline.compute(ReducedReference) + return amor_pipeline + + +@pytestmark +def test_from_measurements_tool_concatenates_event_lists( + pipeline_with_1632_reference: sciline.Pipeline, +): + pl = pipeline_with_1632_reference + + run = { + Filename[SampleRun]: list(map(data.amor_run, (1636, 1639, 1641))), + QBins: sc.geomspace( + dim='Q', start=0.062, stop=0.18, num=391, unit='1/angstrom' + ), + DetectorRotation[SampleRun]: sc.scalar(0.140167, unit='rad'), + SampleRotation[SampleRun]: sc.scalar(0.0680678, unit='rad'), + } + results = from_measurements( + pl, + [run], + target=ReflectivityOverQ, + scale_to_overlap=False, + ) + + results2 = [] + for fname in run[Filename[SampleRun]]: + pl.copy() + pl[Filename[SampleRun]] = fname + pl[QBins] = run[QBins] + pl[DetectorRotation[SampleRun]] = run[DetectorRotation[SampleRun]] + pl[SampleRotation[SampleRun]] = run[SampleRotation[SampleRun]] + results2.append(pl.compute(ReflectivityOverQ).hist().data) + + assert_allclose(sum(results2), results[0].hist().data) From 7fb806bf3802c14906d08d6d7f919dcc3750ce70 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 14:07:48 +0200 Subject: [PATCH 08/29] docs: fix --- src/ess/reflectometry/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index c06d6c3b..0ba11287 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -311,9 +311,9 @@ def from_measurements( The domain type(s) to compute for each run. scale_to_overlap: - If not ``None`` the curves will be scaled to overlap. + If ``True`` the curves will be scaled to overlap. If a tuple then this argument will be passed as the ``critical_edge_interval`` - to the ``scale_reflectivity_curves_to_overlap`` function. + argument to the ``scale_reflectivity_curves_to_overlap`` function. Returns --------- From f097188e186670854875395e79500d218ce4555c Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 6 May 2025 14:08:54 +0200 Subject: [PATCH 09/29] tests --- tests/tools_test.py | 78 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/tools_test.py b/tests/tools_test.py index 40b7dc39..7f1c81b5 100644 --- a/tests/tools_test.py +++ b/tests/tools_test.py @@ -14,7 +14,12 @@ linlogspace, scale_reflectivity_curves_to_overlap, ) -from ess.reflectometry.types import Filename, ReflectivityOverQ, SampleRun +from ess.reflectometry.types import ( + Filename, + ReducibleData, + ReflectivityOverQ, + SampleRun, +) def curve(d, qmin, qmax): @@ -268,3 +273,74 @@ def B(x: str) -> int: ) assert len(datasets) == len(params) assert type(datasets) is type(params) + + +def test_from_measurements_tool_does_not_recompute_reflectivity(): + R = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + + times_evaluated = 0 + + def reflectivity() -> ReflectivityOverQ: + nonlocal times_evaluated + times_evaluated += 1 + return ReflectivityOverQ(R) + + def reducible_data() -> ReducibleData[SampleRun]: + return 'Not important' + + pl = sl.Pipeline([reflectivity, reducible_data]) + + from_measurements( + pl, + [{}, {}], + target=(ReflectivityOverQ,), + scale_to_overlap=True, + ) + assert times_evaluated == 2 + + +def test_from_measurements_tool_applies_scaling_to_reflectivityoverq(): + R1 = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + R2 = 0.5 * R1 + + def reducible_data() -> ReducibleData[SampleRun]: + return 'Not important' + + pl = sl.Pipeline([reducible_data]) + + results = from_measurements( + pl, + [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], + target=(ReflectivityOverQ,), + scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), + ) + assert_allclose(results[0][ReflectivityOverQ], results[1][ReflectivityOverQ]) + + +def test_from_measurements_tool_applies_scaling_to_reducibledata(): + R1 = sc.DataArray( + sc.ones(dims=['Q'], shape=(50,), with_variances=True), + coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, + ).bin(Q=10) + R2 = 0.5 * R1 + + def reducible_data() -> ReducibleData[SampleRun]: + return sc.scalar(1) + + pl = sl.Pipeline([reducible_data]) + + results = from_measurements( + pl, + [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], + target=(ReducibleData[SampleRun],), + scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), + ) + assert_allclose( + results[0][ReducibleData[SampleRun]], 0.5 * results[1][ReducibleData[SampleRun]] + ) From f158ac28c4f8826333d214d6aeb5b2cee6dd7788 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 3 Jul 2025 23:30:15 +0200 Subject: [PATCH 10/29] add scaling factor to Amor workflow --- src/ess/amor/__init__.py | 2 ++ src/ess/amor/workflow.py | 18 +++++++++++++++--- src/ess/reflectometry/types.py | 8 ++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/ess/amor/__init__.py b/src/ess/amor/__init__.py index 4148b94b..5c67959f 100644 --- a/src/ess/amor/__init__.py +++ b/src/ess/amor/__init__.py @@ -16,6 +16,7 @@ Position, RunType, SampleRotationOffset, + ScalingFactorForOverlap, ) from . import ( conversions, @@ -74,6 +75,7 @@ def default_parameters() -> dict: ), GravityToggle: True, SampleRotationOffset[RunType]: sc.scalar(0.0, unit='deg'), + ScalingFactorForOverlap[RunType]: 1.0, } diff --git a/src/ess/amor/workflow.py b/src/ess/amor/workflow.py index 828abe4f..68dc0644 100644 --- a/src/ess/amor/workflow.py +++ b/src/ess/amor/workflow.py @@ -12,6 +12,8 @@ ProtonCurrent, ReducibleData, RunType, + ScalingFactorForOverlap, + UnscaledReducibleData, WavelengthBins, YIndexLimits, ZIndexLimits, @@ -27,7 +29,7 @@ def add_coords_masks_and_apply_corrections( wbins: WavelengthBins, proton_current: ProtonCurrent[RunType], graph: CoordTransformationGraph, -) -> ReducibleData[RunType]: +) -> UnscaledReducibleData[RunType]: """ Computes coordinates, masks and corrections that are the same for the sample measurement and the reference measurement. @@ -43,7 +45,17 @@ def add_coords_masks_and_apply_corrections( da = add_proton_current_mask(da) da = correct_by_proton_current(da) - return ReducibleData[RunType](da) + return UnscaledReducibleData[RunType](da) + + +def scale_raw_reducible_data( + da: UnscaledReducibleData[RunType], + scale: ScalingFactorForOverlap[RunType], +) -> ReducibleData[RunType]: + """ + Scales the raw data by a given factor. + """ + return ReducibleData[RunType](da * scale) -providers = (add_coords_masks_and_apply_corrections,) +providers = (add_coords_masks_and_apply_corrections, scale_raw_reducible_data) diff --git a/src/ess/reflectometry/types.py b/src/ess/reflectometry/types.py index f1eea80a..6959534b 100644 --- a/src/ess/reflectometry/types.py +++ b/src/ess/reflectometry/types.py @@ -28,10 +28,18 @@ class RawChopper(sciline.Scope[RunType, sc.DataGroup], sc.DataGroup): """Chopper data loaded from nexus file.""" +class UnscaledReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): + """""" + + class ReducibleData(sciline.Scope[RunType, sc.DataArray], sc.DataArray): """Event data with common coordinates added""" +class ScalingFactorForOverlap(sciline.Scope[RunType, float], float): + """""" + + ReducedReference = NewType("ReducedReference", sc.DataArray) """Intensity distribution on the detector for a sample with :math`R(Q) = 1`""" From 268f48781044fc34fe0f95d91cde1f189c5fa495 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 3 Jul 2025 23:50:06 +0200 Subject: [PATCH 11/29] map on unscaled data --- src/ess/reflectometry/workflow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index 76dfe734..e960da99 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -16,10 +16,10 @@ DetectorRotation, Filename, RawChopper, - ReducibleData, RunType, SampleRotation, SampleRun, + UnscaledReducibleData, ) @@ -64,11 +64,11 @@ def with_filenames( mapped = wf.map(df) try: - wf[ReducibleData[runtype]] = mapped[ReducibleData[runtype]].reduce( - index=axis_name, func=_concatenate_event_lists - ) + wf[UnscaledReducibleData[runtype]] = mapped[ + UnscaledReducibleData[runtype] + ].reduce(index=axis_name, func=_concatenate_event_lists) except ValueError: - # ReducibleData[runtype] is independent of Filename[runtype] + # UnscaledReducibleData[runtype] is independent of Filename[runtype] pass try: wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( From eb56472c8fbba36ba9fdccecc925a86908359235 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 16:18:04 +0200 Subject: [PATCH 12/29] add batch_processor and WorkflowCollection to the tools --- src/ess/reflectometry/__init__.py | 17 +- src/ess/reflectometry/tools.py | 309 ++++++++++++++++++++---------- 2 files changed, 226 insertions(+), 100 deletions(-) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index 7dd54dce..906c8271 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -12,6 +12,7 @@ from . import conversions, corrections, figures, normalization, orso from .load import load_reference, save_reference +from .tools import batch_processor providers = ( *corrections.providers, @@ -31,9 +32,23 @@ del importlib - __all__ = [ + "__version__", + "batch_processor", + "conversions", + "corrections", "figures", "load_reference", + "normalization", + "orso", + "providers", "save_reference", ] + +# __all__ = [ +# "__version__", +# "batch_processor", +# "figures", +# "load_reference", +# "save_reference", +# ] diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index cfbd36a2..4afda64a 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -12,9 +12,12 @@ from ess.reflectometry import orso from ess.reflectometry.types import ( Filename, + QBins, ReducibleData, ReflectivityOverQ, SampleRun, + ScalingFactorForOverlap, + UnscaledReducibleData, ) from ess.reflectometry.workflow import with_filenames @@ -165,44 +168,65 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - curves: Sequence[sc.DataArray], + wf_collection: Sequence[sc.DataArray], critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, + cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: - '''Make the curves overlap by scaling all except the first by a factor. + ''' + Set the ``ScalingFactorForOverlap`` parameter on the provided workflows + in a way that would makes the 1D reflectivity curves overlap. + + If :code:`critical_edge_interval` is not provided, all workflows are scaled except + the data with the lowest Q-range, which is considered to be the reference curve. The scaling factors are determined by a maximum likelihood estimate (assuming the errors are normal distributed). - If :code:`critical_edge_interval` is provided then all curves are scaled. + If :code:`critical_edge_interval` is provided then all data are scaled. - All curves must be have the same unit for data and the Q-coordinate. + All reflectivity curves must be have the same unit for data and the Q-coordinate. Parameters --------- - curves: - the reflectivity curves that should be scaled together + wf_collection: + The collection of workflows that can compute the ``ReflectivityOverQ``. critical_edge_interval: - a tuple denoting an interval that is known to belong + A tuple denoting an interval that is known to belong to the critical edge, i.e. where the reflectivity is known to be 1. + cache_intermediate_results: + If ``True`` the intermediate results ``UnscaledReducibleData`` will be cached + (this is the base for all types that are downstream of the scaling factor). Returns --------- : A list of scaled reflectivity curves and a list of the scaling factors. ''' - if critical_edge_interval is not None: - q = next(iter(curves)).coords['Q'] - N = ( - ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - .sum() - .value + # if critical_edge_interval is not None: + # # Find q bins with the lowest Q start point + # qmin = min( + # wf.compute(QBins).min() for wf in wf_collection.values()) + # q = next(iter(wf_collection.values())).compute(QBins) + # N = ( + # ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) + # .sum() + # .value + # ) + # edge = sc.DataArray( + # data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), + # coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, + # ) + # curves, factors = scale_reflectivity_curves_to_overlap([edge, *curves]) + # return curves[1:], factors[1:] + + wfc = wf_collection.copy() + if cache_intermediate_results: + wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( + UnscaledReducibleData[SampleRun] ) - edge = sc.DataArray( - data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - ) - curves, factors = scale_reflectivity_curves_to_overlap([edge, *curves]) - return curves[1:], factors[1:] + + curves = {key: r.hist() for key, r in wfc.compute(ReflectivityOverQ).items()} + if len({c.data.unit for c in curves}) != 1: raise ValueError('The reflectivity curves must have the same unit') if len({c.coords['Q'].unit for c in curves}) != 1: @@ -226,10 +250,13 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - return [ - scaling_factor * curve - for scaling_factor, curve in zip(scaling_factors, curves, strict=True) - ], scaling_factors + + wfc[ScalingFactorForOverlap[SampleRun]] = scaling_factors + return wfc + # return [ + # scaling_factor * curve + # for scaling_factor, curve in zip(scaling_factors, curves, strict=True) + # ], scaling_factors def combine_curves( @@ -284,44 +311,71 @@ def combine_curves( ) -def from_measurements( - workflow: sciline.Pipeline, - runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], - target: type | Sequence[type] = orso.OrsoIofQDataset, - *, - scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, -) -> list | Mapping: - '''Produces a list of datasets containing for each of the provided runs. - Each entry of :code:`runs` is a mapping of parameters and - values needed to produce the dataset. +class WorkflowCollection: + """ + A collection of sciline workflows that can be used to compute multiple + targets from multiple workflows. + It can also be used to set parameters for all workflows in a single shot. + """ - Optionally, the results can be scaled so that the reflectivity curves overlap in - the regions where they have the same Q-value. + def __init__(self, workflows: Mapping[str, sciline.Pipeline]): + self._workflows = {name: pl.copy() for name, pl in workflows.items()} - Parameters - ----------- - workflow: - The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. + def __setitem__(self, key: type, value: Any | Mapping[type, Any]): + if hasattr(value, 'items'): + for name, v in value.items(): + self._workflows[name][key] = v + else: + for pl in self._workflows.values(): + pl[key] = value - runs: - The sciline parameters to be used for each run. + def compute(self, target: type | Sequence[type]) -> Mapping[str, Any]: + return {name: pl.compute(target) for name, pl in self._workflows.items()} - target: - The domain type(s) to compute for each run. + def copy(self) -> 'WorkflowCollection': + return self.__class__(self._workflows) - scale_to_overlap: - If ``True`` the curves will be scaled to overlap. - If a tuple then this argument will be passed as the ``critical_edge_interval`` - argument to the ``scale_reflectivity_curves_to_overlap`` function. + def keys(self) -> Sequence[str]: + return self._workflows.keys() - Returns - --------- - list of the computed ORSO datasets, containing one reflectivity curve each - ''' - names = runs.keys() if hasattr(runs, 'keys') else None - runs = runs.values() if hasattr(runs, 'values') else runs + def values(self) -> Sequence[sciline.Pipeline]: + return self._workflows.values() - def init_workflow(workflow, parameters): + def items(self) -> Sequence[tuple[str, sciline.Pipeline]]: + return self._workflows.items() + + def add(self, name: str, workflow: sciline.Pipeline): + """ + Adds a new workflow to the collection. + """ + self._workflows[name] = workflow.copy() + + def remove(self, name: str): + """ + Removes a workflow from the collection by its name. + """ + del self._workflows[name] + + +def batch_processor( + workflow: sciline.Pipeline, runs: Mapping[Any, Mapping[type, Any]] +) -> WorkflowCollection: + """ + Creates a collection of sciline workflows from the provided runs. + + Runs can be provided as a mapping of names to parameters or as a sequence + of mappings of parameters and values. + + Parameters + ---------- + workflow: + The sciline workflow used to compute the targets for each of the runs. + runs: + The sciline parameters to be used for each run. + TODO: explain how grouping works depending on the type of `runs`. + """ + workflows = {} + for name, parameters in runs.items(): wf = workflow.copy() for tp, value in parameters.items(): if tp is Filename[SampleRun]: @@ -337,48 +391,105 @@ def init_workflow(workflow, parameters): ) else: wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] - return wf - - if scale_to_overlap: - results = [] - for parameters in runs: - wf = init_workflow(workflow, parameters) - results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) - - scale_factors = scale_reflectivity_curves_to_overlap( - [r[ReflectivityOverQ].hist() for r in results], - critical_edge_interval=scale_to_overlap - if isinstance(scale_to_overlap, list | tuple) - else None, - )[1] - - datasets = [] - for i, parameters in enumerate(runs): - wf = init_workflow(workflow, parameters) - if scale_to_overlap: - # Optimization in case we can avoid re-doing some work: - # Check if any of the targets need ReducibleData if - # ReflectivityOverQ already exists. - # If they don't, we can avoid recomputing ReducibleData. - targets = target if hasattr(target, '__len__') else (target,) - if any( - _workflow_needs_quantity_A_even_if_quantity_B_is_set( - wf[t], ReducibleData[SampleRun], ReflectivityOverQ - ) - for t in targets - ): - wf[ReducibleData[SampleRun]] = ( - scale_factors[i] * results[i][ReducibleData[SampleRun]] - ) - else: - wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] - - dataset = wf.compute(target) - datasets.append(dataset) - return datasets if names is None else dict(zip(names, datasets, strict=True)) - - -def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): - wf = workflow.copy() - wf[B] = 'Not important' - return A in wf.underlying_graph + workflows[name] = wf + return WorkflowCollection(workflows) + + +# def from_measurements( +# workflow: sciline.Pipeline, +# runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], +# target: type | Sequence[type] = orso.OrsoIofQDataset, +# *, +# scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, +# ) -> list | Mapping: +# '''Produces a list of datasets containing for each of the provided runs. +# Each entry of :code:`runs` is a mapping of parameters and +# values needed to produce the dataset. + +# Optionally, the results can be scaled so that the reflectivity curves overlap in +# the regions where they have the same Q-value. + +# Parameters +# ----------- +# workflow: +# The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. + +# runs: +# The sciline parameters to be used for each run. + +# target: +# The domain type(s) to compute for each run. + +# scale_to_overlap: +# If ``True`` the curves will be scaled to overlap. +# If a tuple then this argument will be passed as the ``critical_edge_interval`` +# argument to the ``scale_reflectivity_curves_to_overlap`` function. + +# Returns +# --------- +# list of the computed ORSO datasets, containing one reflectivity curve each +# ''' +# names = runs.keys() if hasattr(runs, 'keys') else None +# runs = runs.values() if hasattr(runs, 'values') else runs + +# def init_workflow(workflow, parameters): +# wf = workflow.copy() +# for tp, value in parameters.items(): +# if tp is Filename[SampleRun]: +# continue +# wf[tp] = value + +# if Filename[SampleRun] in parameters: +# if isinstance(parameters[Filename[SampleRun]], list | tuple): +# wf = with_filenames( +# wf, +# SampleRun, +# parameters[Filename[SampleRun]], +# ) +# else: +# wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] +# return wf + +# if scale_to_overlap: +# results = [] +# for parameters in runs: +# wf = init_workflow(workflow, parameters) +# results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) + +# scale_factors = scale_reflectivity_curves_to_overlap( +# [r[ReflectivityOverQ].hist() for r in results], +# critical_edge_interval=scale_to_overlap +# if isinstance(scale_to_overlap, list | tuple) +# else None, +# )[1] + +# datasets = [] +# for i, parameters in enumerate(runs): +# wf = init_workflow(workflow, parameters) +# if scale_to_overlap: +# # Optimization in case we can avoid re-doing some work: +# # Check if any of the targets need ReducibleData if +# # ReflectivityOverQ already exists. +# # If they don't, we can avoid recomputing ReducibleData. +# targets = target if hasattr(target, '__len__') else (target,) +# if any( +# _workflow_needs_quantity_A_even_if_quantity_B_is_set( +# wf[t], ReducibleData[SampleRun], ReflectivityOverQ +# ) +# for t in targets +# ): +# wf[ReducibleData[SampleRun]] = ( +# scale_factors[i] * results[i][ReducibleData[SampleRun]] +# ) +# else: +# wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] + +# dataset = wf.compute(target) +# datasets.append(dataset) +# return datasets if names is None else dict(zip(names, datasets, strict=True)) + + +# def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): +# wf = workflow.copy() +# wf[B] = 'Not important' +# return A in wf.underlying_graph From 9637acd5a26bc5e5345dee416d96958a93f89afd Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 17:13:54 +0200 Subject: [PATCH 13/29] udpate Amor notebook to use workflow collection --- docs/user-guide/amor/amor-reduction.ipynb | 127 ++++++++-------------- src/ess/reflectometry/tools.py | 123 +++------------------ 2 files changed, 63 insertions(+), 187 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index c3918f94..c3293114 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -30,11 +30,12 @@ "from ess.amor import data # noqa: F401\n", "from ess.reflectometry.types import *\n", "from ess.amor.types import *\n", + "from ess.reflectometry import batch_processor\n", "\n", "# The files used in this tutorial have some issues that makes scippnexus\n", "# raise warnings when loading them. To avoid noise in the notebook the warnings are silenced.\n", "warnings.filterwarnings('ignore', 'Failed to convert .* into a transformation')\n", - "warnings.filterwarnings('ignore', 'Invalid transformation, missing attribute')" + "warnings.filterwarnings('ignore', 'Invalid transformation')" ] }, { @@ -124,19 +125,17 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "## Computing sample reflectivity\n", + "## Computing sample reflectivity from batch reduction\n", "\n", "We now compute the sample reflectivity from 4 runs that used different sample rotation angles.\n", - "The measurements at different rotation angles cover different ranges of $Q$." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ + "The measurements at different rotation angles cover different ranges of $Q$.\n", + "\n", + "We set up a batch reduction helper (using the `batch_processor` function) which makes it easy to process multiple runs at once.\n", + "\n", "In this tutorial we use some Amor data files we have received.\n", "The file paths to the tutorial files are obtained by calling:" ] @@ -184,15 +183,15 @@ " },\n", "}\n", "\n", + "batch = batch_processor(workflow, runs)\n", + "display(batch.keys())\n", "\n", - "reflectivity = {}\n", - "for run_number, params in runs.items():\n", - " wf = workflow.copy()\n", - " for key, value in params.items():\n", - " wf[key] = value\n", - " reflectivity[run_number] = wf.compute(ReflectivityOverQ).hist()\n", - "\n", - "sc.plot(reflectivity, norm='log', vmin=1e-4)" + "# Compute R(Q) for all runs\n", + "reflectivity = batch.compute(ReflectivityOverQ)\n", + "sc.plot(\n", + " {key: r.hist() for key, r in reflectivity.items()},\n", + " norm='log', vmin=1e-4\n", + ")" ] }, { @@ -212,13 +211,17 @@ "source": [ "from ess.reflectometry.tools import scale_reflectivity_curves_to_overlap\n", "\n", - "scaled_reflectivity_curves, scale_factors = scale_reflectivity_curves_to_overlap(\n", - " reflectivity.values(),\n", - " # Optionally specify a Q-interval where the reflectivity is known to be 1.0\n", - " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", + "# Pass the batch workflow collection and get a new workflow collection as output,\n", + "# with the correct scaling factors applied.\n", + "scaled_wf = scale_reflectivity_curves_to_overlap(\n", + " batch,\n", + " # TODO: implement handling of the critical_edge_interval\n", + " # critical_edge_interval=sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom')\n", ")\n", "\n", - "sc.plot(dict(zip(reflectivity.keys(), scaled_reflectivity_curves, strict=True)), norm='log', vmin=1e-5)" + "scaled_r = {key: r.hist() for key, r in scaled_wf.compute(ReflectivityOverQ).items()}\n", + "\n", + "sc.plot(scaled_r, norm='log', vmin=1e-5)" ] }, { @@ -235,7 +238,7 @@ "outputs": [], "source": [ "from ess.reflectometry.tools import combine_curves\n", - "combined = combine_curves(scaled_reflectivity_curves, workflow.compute(QBins))\n", + "combined = combine_curves(scaled_r.values(), workflow.compute(QBins))\n", "combined.plot(norm='log')" ] }, @@ -265,26 +268,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Start by computing the `ReflectivityData` for each of the files\n", - "diagnostics = {}\n", - "for run_number, params in runs.items():\n", - " wf = workflow.copy()\n", - " for key, value in params.items():\n", - " wf[key] = value\n", - " diagnostics[run_number] = wf.compute((ReflectivityOverZW, ThetaBins[SampleRun]))\n", - "\n", - "# Scale the results using the scale factors computed earlier\n", - "for run_number, scale_factor in zip(reflectivity.keys(), scale_factors, strict=True):\n", - " diagnostics[run_number][ReflectivityOverZW] *= scale_factor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "diagnostics['608'][ReflectivityOverZW].hist().flatten(('blade', 'wire'), to='z').plot(norm='log')" + "diagnostics = scaled_wf.compute(ReflectivityOverZW)\n", + "diagnostics['608'].hist().flatten(('blade', 'wire'), to='z').plot(norm='log')" ] }, { @@ -304,8 +289,8 @@ "from ess.reflectometry.figures import wavelength_theta_figure\n", "\n", "wavelength_theta_figure(\n", - " [result[ReflectivityOverZW] for result in diagnostics.values()],\n", - " theta_bins=[result[ThetaBins[SampleRun]] for result in diagnostics.values()],\n", + " diagnostics.values(),\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", " q_edges_to_display=(sc.scalar(0.018, unit='1/angstrom'), sc.scalar(0.113, unit='1/angstrom'))\n", ")" ] @@ -334,8 +319,8 @@ "from ess.reflectometry.figures import q_theta_figure\n", "\n", "q_theta_figure(\n", - " [res[ReflectivityOverZW] for res in diagnostics.values()],\n", - " theta_bins=[res[ThetaBins[SampleRun]] for res in diagnostics.values()],\n", + " diagnostics.values(),\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", " q_bins=workflow.compute(QBins)\n", ")" ] @@ -380,8 +365,7 @@ "We can save the computed $I(Q)$ to an [ORSO](https://www.reflectometry.org) [.ort](https://github.com/reflectivity/file_format/blob/master/specification.md) file using the [orsopy](https://orsopy.readthedocs.io/en/latest/index.html) package.\n", "\n", "First, we need to collect the metadata for that file.\n", - "To this end, we build a pipeline with additional providers.\n", - "We also insert a parameter to indicate the creator of the processed data." + "To this end, we insert a parameter to indicate the creator of the processed data." ] }, { @@ -400,7 +384,7 @@ "metadata": {}, "outputs": [], "source": [ - "workflow[orso.OrsoCreator] = orso.OrsoCreator(\n", + "scaled_wf[orso.OrsoCreator] = orso.OrsoCreator(\n", " fileio.base.Person(\n", " name='Max Mustermann',\n", " affiliation='European Spallation Source ERIC',\n", @@ -409,20 +393,11 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "workflow.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We build our ORSO dataset from the computed $I(Q)$ and the ORSO metadata:" + "We can visualize the workflow for a single run (`'608'`):" ] }, { @@ -431,15 +406,14 @@ "metadata": {}, "outputs": [], "source": [ - "iofq_dataset = workflow.compute(orso.OrsoIofQDataset)\n", - "iofq_dataset" + "scaled_wf['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We also add the URL of this notebook to make it easier to reproduce the data:" + "We build our ORSO dataset from the computed $I(Q)$ and the ORSO metadata:" ] }, { @@ -448,17 +422,14 @@ "metadata": {}, "outputs": [], "source": [ - "iofq_dataset.info.reduction.script = (\n", - " 'https://scipp.github.io/essreflectometry/examples/amor.html'\n", - ")" + "iofq_datasets = scaled_wf.compute(orso.OrsoIofQDataset)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Now let's repeat this for all the sample measurements!\n", - "To do that we can use an utility in `ess.reflectometry.tools`:" + "We also add the URL of this notebook to make it easier to reproduce the data:" ] }, { @@ -467,16 +438,10 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.reflectometry.tools import from_measurements\n", - "\n", - "datasets = from_measurements(\n", - " workflow,\n", - " runs.values(),\n", - " target=orso.OrsoIofQDataset,\n", - " # Optionally scale the curves to overlap using `scale_reflectivity_curves_to_overlap`\n", - " # with the same \"critical edge interval\" that was used before\n", - " scale_to_overlap=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", - ")" + "for ds in iofq_datasets.values():\n", + " ds.info.reduction.script = (\n", + " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction.html'\n", + " )" ] }, { @@ -484,7 +449,7 @@ "metadata": {}, "source": [ "Finally, we can save the data to a file.\n", - "Note that `iofq_dataset` is an [orsopy.fileio.orso.OrsoDataset](https://orsopy.readthedocs.io/en/latest/orsopy.fileio.orso.html#orsopy.fileio.orso.OrsoDataset)." + "Note that `iofq_datasets` contains [orsopy.fileio.orso.OrsoDataset](https://orsopy.readthedocs.io/en/latest/orsopy.fileio.orso.html#orsopy.fileio.orso.OrsoDataset)s." ] }, { @@ -493,7 +458,7 @@ "metadata": {}, "outputs": [], "source": [ - "fileio.orso.save_orso(datasets=datasets, fname='amor_reduced_iofq.ort')" + "fileio.orso.save_orso(datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort')" ] }, { @@ -529,7 +494,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 4afda64a..7092a550 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -9,11 +9,11 @@ import scipp as sc import scipy.optimize as opt -from ess.reflectometry import orso +# from ess.reflectometry import orso from ess.reflectometry.types import ( Filename, QBins, - ReducibleData, + # ReducibleData, ReflectivityOverQ, SampleRun, ScalingFactorForOverlap, @@ -225,7 +225,7 @@ def scale_reflectivity_curves_to_overlap( UnscaledReducibleData[SampleRun] ) - curves = {key: r.hist() for key, r in wfc.compute(ReflectivityOverQ).items()} + curves = [r.hist() for r in wfc.compute(ReflectivityOverQ).values()] if len({c.data.unit for c in curves}) != 1: raise ValueError('The reflectivity curves must have the same unit') @@ -251,7 +251,10 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - wfc[ScalingFactorForOverlap[SampleRun]] = scaling_factors + wfc[ScalingFactorForOverlap[SampleRun]] = dict( + zip(wfc.keys(), scaling_factors, strict=True) + ) + return wfc # return [ # scaling_factor * curve @@ -329,8 +332,16 @@ def __setitem__(self, key: type, value: Any | Mapping[type, Any]): for pl in self._workflows.values(): pl[key] = value - def compute(self, target: type | Sequence[type]) -> Mapping[str, Any]: - return {name: pl.compute(target) for name, pl in self._workflows.items()} + def __getitem__(self, name: str) -> sciline.Pipeline: + """ + Returns a single workflow from the collection given by its name. + """ + return self._workflows[name] + + def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + return { + name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() + } def copy(self) -> 'WorkflowCollection': return self.__class__(self._workflows) @@ -393,103 +404,3 @@ def batch_processor( wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] workflows[name] = wf return WorkflowCollection(workflows) - - -# def from_measurements( -# workflow: sciline.Pipeline, -# runs: Sequence[Mapping[type, Any]] | Mapping[Any, Mapping[type, Any]], -# target: type | Sequence[type] = orso.OrsoIofQDataset, -# *, -# scale_to_overlap: bool | tuple[sc.Variable, sc.Variable] = False, -# ) -> list | Mapping: -# '''Produces a list of datasets containing for each of the provided runs. -# Each entry of :code:`runs` is a mapping of parameters and -# values needed to produce the dataset. - -# Optionally, the results can be scaled so that the reflectivity curves overlap in -# the regions where they have the same Q-value. - -# Parameters -# ----------- -# workflow: -# The sciline workflow used to compute `ReflectivityOverQ` for each of the runs. - -# runs: -# The sciline parameters to be used for each run. - -# target: -# The domain type(s) to compute for each run. - -# scale_to_overlap: -# If ``True`` the curves will be scaled to overlap. -# If a tuple then this argument will be passed as the ``critical_edge_interval`` -# argument to the ``scale_reflectivity_curves_to_overlap`` function. - -# Returns -# --------- -# list of the computed ORSO datasets, containing one reflectivity curve each -# ''' -# names = runs.keys() if hasattr(runs, 'keys') else None -# runs = runs.values() if hasattr(runs, 'values') else runs - -# def init_workflow(workflow, parameters): -# wf = workflow.copy() -# for tp, value in parameters.items(): -# if tp is Filename[SampleRun]: -# continue -# wf[tp] = value - -# if Filename[SampleRun] in parameters: -# if isinstance(parameters[Filename[SampleRun]], list | tuple): -# wf = with_filenames( -# wf, -# SampleRun, -# parameters[Filename[SampleRun]], -# ) -# else: -# wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] -# return wf - -# if scale_to_overlap: -# results = [] -# for parameters in runs: -# wf = init_workflow(workflow, parameters) -# results.append(wf.compute((ReflectivityOverQ, ReducibleData[SampleRun]))) - -# scale_factors = scale_reflectivity_curves_to_overlap( -# [r[ReflectivityOverQ].hist() for r in results], -# critical_edge_interval=scale_to_overlap -# if isinstance(scale_to_overlap, list | tuple) -# else None, -# )[1] - -# datasets = [] -# for i, parameters in enumerate(runs): -# wf = init_workflow(workflow, parameters) -# if scale_to_overlap: -# # Optimization in case we can avoid re-doing some work: -# # Check if any of the targets need ReducibleData if -# # ReflectivityOverQ already exists. -# # If they don't, we can avoid recomputing ReducibleData. -# targets = target if hasattr(target, '__len__') else (target,) -# if any( -# _workflow_needs_quantity_A_even_if_quantity_B_is_set( -# wf[t], ReducibleData[SampleRun], ReflectivityOverQ -# ) -# for t in targets -# ): -# wf[ReducibleData[SampleRun]] = ( -# scale_factors[i] * results[i][ReducibleData[SampleRun]] -# ) -# else: -# wf[ReflectivityOverQ] = scale_factors[i] * results[i][ReflectivityOverQ] - -# dataset = wf.compute(target) -# datasets.append(dataset) -# return datasets if names is None else dict(zip(names, datasets, strict=True)) - - -# def _workflow_needs_quantity_A_even_if_quantity_B_is_set(workflow, A, B): -# wf = workflow.copy() -# wf[B] = 'Not important' -# return A in wf.underlying_graph From 2d2871dc960a598232d1bd4d0c324efb9af7c62a Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 18:09:06 +0200 Subject: [PATCH 14/29] fix critical_edge parameter in scaling --- docs/user-guide/amor/amor-reduction.ipynb | 3 +- src/ess/reflectometry/tools.py | 78 ++++++++++++++--------- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index c3293114..18d9224e 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -215,8 +215,7 @@ "# with the correct scaling factors applied.\n", "scaled_wf = scale_reflectivity_curves_to_overlap(\n", " batch,\n", - " # TODO: implement handling of the critical_edge_interval\n", - " # critical_edge_interval=sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom')\n", + " critical_edge_interval=(sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'))\n", ")\n", "\n", "scaled_r = {key: r.hist() for key, r in scaled_wf.compute(ReflectivityOverQ).items()}\n", diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 7092a550..89114ee1 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -2,7 +2,7 @@ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) from collections.abc import Mapping, Sequence from itertools import chain -from typing import Any +from typing import Any, NewType import numpy as np import sciline @@ -13,7 +13,6 @@ from ess.reflectometry.types import ( Filename, QBins, - # ReducibleData, ReflectivityOverQ, SampleRun, ScalingFactorForOverlap, @@ -167,6 +166,10 @@ def _interpolate_on_qgrid(curves, grid): ) +CriticalEdgeKey = NewType('CriticalEdgeKey', None) +"""A unique key used to store a 'fake' critical edge in a workflow collection.""" + + def scale_reflectivity_curves_to_overlap( wf_collection: Sequence[sc.DataArray], critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, @@ -202,22 +205,29 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - # if critical_edge_interval is not None: - # # Find q bins with the lowest Q start point - # qmin = min( - # wf.compute(QBins).min() for wf in wf_collection.values()) - # q = next(iter(wf_collection.values())).compute(QBins) - # N = ( - # ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - # .sum() - # .value - # ) - # edge = sc.DataArray( - # data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - # coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - # ) - # curves, factors = scale_reflectivity_curves_to_overlap([edge, *curves]) - # return curves[1:], factors[1:] + if critical_edge_interval is not None: + # Find q bins with the lowest Q start point + q = min( + (wf.compute(QBins) for wf in wf_collection.values()), + key=lambda q_: q_.min(), + ) + N = ( + ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) + .sum() + .value + ) + edge = sc.DataArray( + data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), + coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, + ) + wfc = wf_collection.copy() + underlying_wf = next(iter(wfc.values())) + edge_wf = underlying_wf.copy() + edge_wf[ReflectivityOverQ] = edge + wfc.add(CriticalEdgeKey, edge_wf) + return scale_reflectivity_curves_to_overlap( + wfc, cache_intermediate_results=cache_intermediate_results + ) wfc = wf_collection.copy() if cache_intermediate_results: @@ -225,17 +235,28 @@ def scale_reflectivity_curves_to_overlap( UnscaledReducibleData[SampleRun] ) - curves = [r.hist() for r in wfc.compute(ReflectivityOverQ).values()] + reflectivities = wfc.compute(ReflectivityOverQ) - if len({c.data.unit for c in curves}) != 1: + # First sort the dict of reflectivities by the Q min value + curves = { + k: v.hist() if v.bins is not None else v + for k, v in sorted( + reflectivities.items(), key=lambda item: item[1].coords['Q'].min().value + ) + } + # Now place the critical edge at the beginning, if it exists + if CriticalEdgeKey in curves.keys(): + curves = {CriticalEdgeKey: curves[CriticalEdgeKey]} | curves + + if len({c.data.unit for c in curves.values()}) != 1: raise ValueError('The reflectivity curves must have the same unit') - if len({c.coords['Q'].unit for c in curves}) != 1: + if len({c.coords['Q'].unit for c in curves.values()}) != 1: raise ValueError('The Q-coordinates must have the same unit for each curve') - qgrid = _create_qgrid_where_overlapping([c.coords['Q'] for c in curves]) + qgrid = _create_qgrid_where_overlapping([c.coords['Q'] for c in curves.values()]) - r = _interpolate_on_qgrid(map(sc.values, curves), qgrid).values - v = _interpolate_on_qgrid(map(sc.variances, curves), qgrid).values + r = _interpolate_on_qgrid(map(sc.values, curves.values()), qgrid).values + v = _interpolate_on_qgrid(map(sc.variances, curves.values()), qgrid).values def cost(scaling_factors): scaling_factors = np.concatenate([[1.0], scaling_factors])[:, None] @@ -252,14 +273,13 @@ def cost(scaling_factors): scaling_factors = (1.0, *map(float, sol.x)) wfc[ScalingFactorForOverlap[SampleRun]] = dict( - zip(wfc.keys(), scaling_factors, strict=True) + zip(curves.keys(), scaling_factors, strict=True) ) + if CriticalEdgeKey in wfc.keys(): + wfc.remove(CriticalEdgeKey) + return wfc - # return [ - # scaling_factor * curve - # for scaling_factor, curve in zip(scaling_factors, curves, strict=True) - # ], scaling_factors def combine_curves( From 2a5120dc559608aaacd54fdb53a77f61db6194a2 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 15 Jul 2025 18:47:16 +0200 Subject: [PATCH 15/29] remove commented code --- src/ess/reflectometry/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/ess/reflectometry/__init__.py b/src/ess/reflectometry/__init__.py index 906c8271..3b6d18b2 100644 --- a/src/ess/reflectometry/__init__.py +++ b/src/ess/reflectometry/__init__.py @@ -44,11 +44,3 @@ "providers", "save_reference", ] - -# __all__ = [ -# "__version__", -# "batch_processor", -# "figures", -# "load_reference", -# "save_reference", -# ] From aced05051ea456b752b588cfb9024e02aaa80a3d Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 5 Aug 2025 15:59:35 +0200 Subject: [PATCH 16/29] add tests for wf collection --- tests/{ => reflectometry}/corrections_test.py | 0 tests/{ => reflectometry}/orso_test.py | 0 tests/{ => reflectometry}/tools_test.py | 0 .../reflectometry/workflow_collection_test.py | 136 ++++++++++++++++++ 4 files changed, 136 insertions(+) rename tests/{ => reflectometry}/corrections_test.py (100%) rename tests/{ => reflectometry}/orso_test.py (100%) rename tests/{ => reflectometry}/tools_test.py (100%) create mode 100644 tests/reflectometry/workflow_collection_test.py diff --git a/tests/corrections_test.py b/tests/reflectometry/corrections_test.py similarity index 100% rename from tests/corrections_test.py rename to tests/reflectometry/corrections_test.py diff --git a/tests/orso_test.py b/tests/reflectometry/orso_test.py similarity index 100% rename from tests/orso_test.py rename to tests/reflectometry/orso_test.py diff --git a/tests/tools_test.py b/tests/reflectometry/tools_test.py similarity index 100% rename from tests/tools_test.py rename to tests/reflectometry/tools_test.py diff --git a/tests/reflectometry/workflow_collection_test.py b/tests/reflectometry/workflow_collection_test.py new file mode 100644 index 00000000..bb1ff529 --- /dev/null +++ b/tests/reflectometry/workflow_collection_test.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) + +import sciline as sl + +from ess.reflectometry.tools import WorkflowCollection + + +def int_to_float(x: int) -> float: + return 0.5 * x + + +def int_float_to_str(x: int, y: float) -> str: + return f"{x};{y}" + + +def test_compute() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + assert coll.compute(float) == {'a': 1.5, 'b': 2.0} + assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + + +def test_compute_multiple() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + result = coll.compute([float, str]) + + assert result['a'] == {float: 1.5, str: '3;1.5'} + assert result['b'] == {float: 2.0, str: '4;2.0'} + + +def test_setitem_mapping() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll[int] = {'a': 7, 'b': 8} + + assert coll.compute(float) == {'a': 3.5, 'b': 4.0} + assert coll.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} + + +def test_setitem_single_value() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll[int] = 5 + + assert coll.compute(float) == {'a': 2.5, 'b': 2.5} + assert coll.compute(str) == {'a': '5;2.5', 'b': '5;2.5'} + + +def test_copy() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll_copy = coll.copy() + + assert coll_copy.compute(float) == {'a': 1.5, 'b': 2.0} + assert coll_copy.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + + coll_copy[int] = {'a': 7, 'b': 8} + assert coll.compute(float) == {'a': 1.5, 'b': 2.0} + assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + assert coll_copy.compute(float) == {'a': 3.5, 'b': 4.0} + assert coll_copy.compute(str) == {'a': '7;3.5', 'b': '8;4.0'} + + +def test_add_workflow() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + wfc = wf.copy() + wfc[int] = 5 + coll.add('c', wfc) + + assert coll.compute(float) == {'a': 1.5, 'b': 2.0, 'c': 2.5} + assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0', 'c': '5;2.5'} + + +def test_add_workflow_with_existing_key() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + wfc = wf.copy() + wfc[int] = 5 + coll.add('a', wfc) + + assert coll.compute(float) == {'a': 2.5, 'b': 2.0} + assert coll.compute(str) == {'a': '5;2.5', 'b': '4;2.0'} + assert 'c' not in coll.keys() # 'c' should not exist + + +def test_remove_workflow() -> None: + wf = sl.Pipeline([int_to_float, int_float_to_str]) + wfa = wf.copy() + wfa[int] = 3 + wfb = wf.copy() + wfb[int] = 4 + coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll.remove('b') + + assert 'b' not in coll.keys() + assert coll.compute(float) == {'a': 1.5} + assert coll.compute(str) == {'a': '3;1.5'} From 504405bae9e1be4707be4789d5a92b655fe67a07 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 5 Aug 2025 16:08:22 +0200 Subject: [PATCH 17/29] improve docstring --- src/ess/reflectometry/tools.py | 38 +++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 89114ee1..dfc594b9 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -394,8 +394,36 @@ def batch_processor( """ Creates a collection of sciline workflows from the provided runs. - Runs can be provided as a mapping of names to parameters or as a sequence - of mappings of parameters and values. + Example: + + ``` + from ess.reflectometry import amor, tools + + workflow = amor.AmorWorkflow() + + runs = { + '608': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(608), + }, + '609': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(609), + }, + '610': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(610), + }, + '611': { + SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + Filename[SampleRun]: amor.data.amor_run(611), + }, + } + + batch = tools.batch_processor(workflow, runs) + + results = batch.compute(ReflectivityOverQ) + ``` Parameters ---------- @@ -403,7 +431,11 @@ def batch_processor( The sciline workflow used to compute the targets for each of the runs. runs: The sciline parameters to be used for each run. - TODO: explain how grouping works depending on the type of `runs`. + Should be a mapping where the keys are the names of the runs + and the values are mappings of type to value pairs. + In addition, if one of the values for ``Filename[SampleRun]`` + is a list or a tuple, then the events from the files + will be concatenated into a single event list. """ workflows = {} for name, parameters in runs.items(): From 7e0c24d15291a7305b1a0f0a41700fdbd699955b Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 6 Aug 2025 19:08:48 +0200 Subject: [PATCH 18/29] start fixing existing unit tests --- tests/reflectometry/tools_test.py | 102 ++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 18 deletions(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 7f1c81b5..90e78f2d 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -9,41 +9,107 @@ from scipp.testing import assert_allclose from ess.reflectometry.tools import ( + WorkflowCollection, + batch_processor, combine_curves, - from_measurements, linlogspace, scale_reflectivity_curves_to_overlap, ) from ess.reflectometry.types import ( Filename, ReducibleData, + ReferenceRun, ReflectivityOverQ, + RunType, SampleRun, + ScalingFactorForOverlap, + UnscaledReducibleData, ) - -def curve(d, qmin, qmax): - return sc.DataArray(data=d, coords={'Q': sc.linspace('Q', qmin, qmax, len(d) + 1)}) +# def curve(d, qmin, qmax): +# return sc.DataArray(data=d, coords={'Q': sc.linspace('Q', qmin, qmax, len(d) + 1)}) + + +def make_sample_events(scale, qmin, qmax): + n1 = 10 + n2 = 15 + qbins = sc.linspace('Q', qmin, qmax, n1 + n2 + 1) + data = sc.DataArray( + data=sc.concat( + ( + sc.ones(dims=['Q'], shape=[10], with_variances=True), + 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), + ), + dim='Q', + ) + * scale, + coords={'Q': sc.midpoints(qbins, 'Q')}, + ) + data.variances[:] = 0.1 + return data.bin(Q=qbins) -def test_reflectivity_curve_scaling(): - data = sc.concat( - ( - sc.ones(dims=['Q'], shape=[10], with_variances=True), - 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - ), - dim='Q', +def make_reference_events(qmin, qmax): + n = 25 + qbins = sc.linspace('Q', qmin, qmax, n + 1) + data = sc.DataArray( + data=sc.ones(dims=['Q'], shape=[n], with_variances=True), + coords={'Q': sc.midpoints(qbins, 'Q')}, ) data.variances[:] = 0.1 + return data.bin(Q=qbins) - curves, factors = scale_reflectivity_curves_to_overlap( - (curve(data, 0, 0.3), curve(0.8 * data, 0.2, 0.7), curve(0.1 * data, 0.6, 1.0)), - ) - assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) - np_assert_allclose((1, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) +def make_workflow(): + def apply_scaling( + da: UnscaledReducibleData[RunType], + scale: ScalingFactorForOverlap[RunType], + ) -> ReducibleData[RunType]: + """ + Scales the raw data by a given factor. + """ + return ReducibleData[RunType](da * scale) + + def reflectivity( + sample: ReducibleData[SampleRun], reference: ReducibleData[ReferenceRun] + ) -> ReflectivityOverQ: + return ReflectivityOverQ(sample.hist() / reference.hist()) + + return sl.Pipeline([apply_scaling, reflectivity]) + + +def test_reflectivity_curve_scaling(): + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + runs = { + 'a': { + UnscaledReducibleData[SampleRun]: make_sample_events(1, 0, 0.3), + UnscaledReducibleData[ReferenceRun]: make_reference_events(0, 0.3), + }, + 'b': { + UnscaledReducibleData[SampleRun]: make_sample_events(0.8, 0.2, 0.7), + UnscaledReducibleData[ReferenceRun]: make_reference_events(0.2, 0.7), + }, + 'c': { + UnscaledReducibleData[SampleRun]: make_sample_events(0.1, 0.6, 1.0), + UnscaledReducibleData[ReferenceRun]: make_reference_events(0.6, 1.0), + }, + } + workflows = {} + for name, params in runs.items(): + workflows[name] = wf.copy() + for key, value in params.items(): + workflows[name][key] = value + wfc = WorkflowCollection(workflows) + + scaled_wf = scale_reflectivity_curves_to_overlap(wfc) + + factors = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + + assert np.isclose(factors['a'], 1.0) + assert np.isclose(factors['b'], 0.5 / 0.8) + assert np.isclose(factors['c'], 0.25 / 0.1) def test_reflectivity_curve_scaling_with_critical_edge(): From 4bd66d34dd9d6464bc8cf1439f897bfc1bcbaba3 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 6 Aug 2025 20:05:23 +0200 Subject: [PATCH 19/29] refactor first tests slightly --- tests/reflectometry/tools_test.py | 97 ++++++++++++++++++------------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 90e78f2d..a620a3f6 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -17,6 +17,7 @@ ) from ess.reflectometry.types import ( Filename, + QBins, ReducibleData, ReferenceRun, ReflectivityOverQ, @@ -71,9 +72,11 @@ def apply_scaling( return ReducibleData[RunType](da * scale) def reflectivity( - sample: ReducibleData[SampleRun], reference: ReducibleData[ReferenceRun] + sample: ReducibleData[SampleRun], + reference: ReducibleData[ReferenceRun], + qbins: QBins, ) -> ReflectivityOverQ: - return ReflectivityOverQ(sample.hist() / reference.hist()) + return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) return sl.Pipeline([apply_scaling, reflectivity]) @@ -82,25 +85,16 @@ def test_reflectivity_curve_scaling(): wf = make_workflow() wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - runs = { - 'a': { - UnscaledReducibleData[SampleRun]: make_sample_events(1, 0, 0.3), - UnscaledReducibleData[ReferenceRun]: make_reference_events(0, 0.3), - }, - 'b': { - UnscaledReducibleData[SampleRun]: make_sample_events(0.8, 0.2, 0.7), - UnscaledReducibleData[ReferenceRun]: make_reference_events(0.2, 0.7), - }, - 'c': { - UnscaledReducibleData[SampleRun]: make_sample_events(0.1, 0.6, 1.0), - UnscaledReducibleData[ReferenceRun]: make_reference_events(0.6, 1.0), - }, - } + params = {'a': (1, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} - for name, params in runs.items(): - workflows[name] = wf.copy() - for key, value in params.items(): - workflows[name][key] = value + for k, v in params.items(): + sample = make_sample_events(*v) + reference = make_reference_events(v[1], v[2]) + workflows[k] = wf.copy() + workflows[k][UnscaledReducibleData[SampleRun]] = sample + workflows[k][UnscaledReducibleData[ReferenceRun]] = reference + workflows[k][QBins] = sample.coords['Q'] + wfc = WorkflowCollection(workflows) scaled_wf = scale_reflectivity_curves_to_overlap(wfc) @@ -113,28 +107,53 @@ def test_reflectivity_curve_scaling(): def test_reflectivity_curve_scaling_with_critical_edge(): - data = sc.concat( - ( - sc.ones(dims=['Q'], shape=[10], with_variances=True), - 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - ), - dim='Q', - ) - data.variances[:] = 0.1 + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} + workflows = {} + for k, v in params.items(): + sample = make_sample_events(*v) + reference = make_reference_events(v[1], v[2]) + workflows[k] = wf.copy() + workflows[k][UnscaledReducibleData[SampleRun]] = sample + workflows[k][UnscaledReducibleData[ReferenceRun]] = reference + workflows[k][QBins] = sample.coords['Q'] - curves, factors = scale_reflectivity_curves_to_overlap( - ( - 2 * curve(data, 0, 0.3), - curve(0.8 * data, 0.2, 0.7), - curve(0.1 * data, 0.6, 1.0), - ), - critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)), + wfc = WorkflowCollection(workflows) + + scaled_wf = scale_reflectivity_curves_to_overlap( + wfc, critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)) ) - assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) - assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) - np_assert_allclose((0.5, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) + factors = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + + assert np.isclose(factors['a'], 0.5) + assert np.isclose(factors['b'], 0.5 / 0.8) + assert np.isclose(factors['c'], 0.25 / 0.1) + + # data = sc.concat( + # ( + # sc.ones(dims=['Q'], shape=[10], with_variances=True), + # 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), + # ), + # dim='Q', + # ) + # data.variances[:] = 0.1 + + # curves, factors = scale_reflectivity_curves_to_overlap( + # ( + # 2 * curve(data, 0, 0.3), + # curve(0.8 * data, 0.2, 0.7), + # curve(0.1 * data, 0.6, 1.0), + # ), + # critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)), + # ) + + # assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) + # assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) + # assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) + # np_assert_allclose((0.5, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) def test_combined_curves(): From 9f1428831e3f73a43de20a1e1df1da876f85c700 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 12:04:41 +0200 Subject: [PATCH 20/29] simplify critical edge handling --- src/ess/reflectometry/tools.py | 63 ++++++++++++++-------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index dfc594b9..739fca52 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import uuid from collections.abc import Mapping, Sequence from itertools import chain from typing import Any, NewType @@ -166,10 +167,6 @@ def _interpolate_on_qgrid(curves, grid): ) -CriticalEdgeKey = NewType('CriticalEdgeKey', None) -"""A unique key used to store a 'fake' critical edge in a workflow collection.""" - - def scale_reflectivity_curves_to_overlap( wf_collection: Sequence[sc.DataArray], critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, @@ -205,30 +202,6 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - if critical_edge_interval is not None: - # Find q bins with the lowest Q start point - q = min( - (wf.compute(QBins) for wf in wf_collection.values()), - key=lambda q_: q_.min(), - ) - N = ( - ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - .sum() - .value - ) - edge = sc.DataArray( - data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - ) - wfc = wf_collection.copy() - underlying_wf = next(iter(wfc.values())) - edge_wf = underlying_wf.copy() - edge_wf[ReflectivityOverQ] = edge - wfc.add(CriticalEdgeKey, edge_wf) - return scale_reflectivity_curves_to_overlap( - wfc, cache_intermediate_results=cache_intermediate_results - ) - wfc = wf_collection.copy() if cache_intermediate_results: wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( @@ -244,9 +217,26 @@ def scale_reflectivity_curves_to_overlap( reflectivities.items(), key=lambda item: item[1].coords['Q'].min().value ) } - # Now place the critical edge at the beginning, if it exists - if CriticalEdgeKey in curves.keys(): - curves = {CriticalEdgeKey: curves[CriticalEdgeKey]} | curves + + critical_edge_key = None + if critical_edge_interval is not None: + critical_edge_key = uuid.uuid4().hex + # Find q bins with the lowest Q start point + q = min( + (wf.compute(QBins) for wf in wf_collection.values()), + key=lambda q_: q_.min(), + ) + N = ( + ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) + .sum() + .value + ) + edge = sc.DataArray( + data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), + coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, + ) + # Now place the critical edge at the beginning + curves = {critical_edge_key: edge} | curves if len({c.data.unit for c in curves.values()}) != 1: raise ValueError('The reflectivity curves must have the same unit') @@ -272,12 +262,11 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - wfc[ScalingFactorForOverlap[SampleRun]] = dict( - zip(curves.keys(), scaling_factors, strict=True) - ) - - if CriticalEdgeKey in wfc.keys(): - wfc.remove(CriticalEdgeKey) + wfc[ScalingFactorForOverlap[SampleRun]] = { + k: v + for k, v in zip(curves.keys(), scaling_factors, strict=True) + if k != critical_edge_key + } return wfc From fff9b4e6def680b623ffc7cd4cc87f2cb2cbe62d Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:11:59 +0200 Subject: [PATCH 21/29] update/fix tools tests --- src/ess/reflectometry/tools.py | 9 +- src/ess/reflectometry/workflow.py | 43 +++-- tests/reflectometry/tools_test.py | 284 +++++++++++++++--------------- 3 files changed, 173 insertions(+), 163 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 739fca52..587686bc 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -3,7 +3,7 @@ import uuid from collections.abc import Mapping, Sequence from itertools import chain -from typing import Any, NewType +from typing import Any import numpy as np import sciline @@ -14,6 +14,7 @@ from ess.reflectometry.types import ( Filename, QBins, + ReferenceRun, ReflectivityOverQ, SampleRun, ScalingFactorForOverlap, @@ -207,6 +208,9 @@ def scale_reflectivity_curves_to_overlap( wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( UnscaledReducibleData[SampleRun] ) + wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( + UnscaledReducibleData[ReferenceRun] + ) reflectivities = wfc.compute(ReflectivityOverQ) @@ -218,9 +222,8 @@ def scale_reflectivity_curves_to_overlap( ) } - critical_edge_key = None + critical_edge_key = uuid.uuid4().hex if critical_edge_interval is not None: - critical_edge_key = uuid.uuid4().hex # Find q bins with the lowest Q start point q = min( (wf.compute(QBins) for wf in wf_collection.values()), diff --git a/src/ess/reflectometry/workflow.py b/src/ess/reflectometry/workflow.py index e960da99..8a203eb5 100644 --- a/src/ess/reflectometry/workflow.py +++ b/src/ess/reflectometry/workflow.py @@ -67,41 +67,50 @@ def with_filenames( wf[UnscaledReducibleData[runtype]] = mapped[ UnscaledReducibleData[runtype] ].reduce(index=axis_name, func=_concatenate_event_lists) - except ValueError: - # UnscaledReducibleData[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # UnscaledReducibleData[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass try: wf[RawChopper[runtype]] = mapped[RawChopper[runtype]].reduce( index=axis_name, func=_any_value ) - except ValueError: - # RawChopper[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # RawChopper[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass try: wf[SampleRotation[runtype]] = mapped[SampleRotation[runtype]].reduce( index=axis_name, func=_any_value ) - except ValueError: - # SampleRotation[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # SampleRotation[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass try: wf[DetectorRotation[runtype]] = mapped[DetectorRotation[runtype]].reduce( index=axis_name, func=_any_value ) - except ValueError: - # DetectorRotation[runtype] is independent of Filename[runtype] + except (ValueError, KeyError): + # DetectorRotation[runtype] is independent of Filename[runtype] or is not + # present in the workflow. pass if runtype is SampleRun: - wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) - wf[OrsoExperiment] = mapped[OrsoExperiment].reduce( - index=axis_name, func=_any_value - ) - wf[OrsoOwner] = mapped[OrsoOwner].reduce(index=axis_name, func=lambda x, *_: x) - wf[OrsoSampleFilenames] = mapped[OrsoSampleFilenames].reduce( + if OrsoSample in wf.underlying_graph: + wf[OrsoSample] = mapped[OrsoSample].reduce(index=axis_name, func=_any_value) + if OrsoExperiment in wf.underlying_graph: + wf[OrsoExperiment] = mapped[OrsoExperiment].reduce( + index=axis_name, func=_any_value + ) + if OrsoOwner in wf.underlying_graph: + wf[OrsoOwner] = mapped[OrsoOwner].reduce( + index=axis_name, func=lambda x, *_: x + ) + if OrsoSampleFilenames in wf.underlying_graph: # When we don't map over filenames # each OrsoSampleFilenames is a list with a single entry. - index=axis_name, - func=_concatenate_lists, - ) + wf[OrsoSampleFilenames] = mapped[OrsoSampleFilenames].reduce( + index=axis_name, func=_concatenate_lists + ) return wf diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index a620a3f6..4e089684 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) + import numpy as np import pytest import sciline as sl import scipp as sc -from numpy.testing import assert_allclose as np_assert_allclose +from numpy.testing import assert_almost_equal from orsopy.fileio import Orso, OrsoDataset from scipp.testing import assert_allclose @@ -27,9 +28,6 @@ UnscaledReducibleData, ) -# def curve(d, qmin, qmax): -# return sc.DataArray(data=d, coords={'Q': sc.linspace('Q', qmin, qmax, len(d) + 1)}) - def make_sample_events(scale, qmin, qmax): n1 = 10 @@ -47,6 +45,7 @@ def make_sample_events(scale, qmin, qmax): coords={'Q': sc.midpoints(qbins, 'Q')}, ) data.variances[:] = 0.1 + data.unit = 'counts' return data.bin(Q=qbins) @@ -58,10 +57,29 @@ def make_reference_events(qmin, qmax): coords={'Q': sc.midpoints(qbins, 'Q')}, ) data.variances[:] = 0.1 + data.unit = 'counts' return data.bin(Q=qbins) +# class RawData(sl.Scope[RunType, sc.DataArray], sc.DataArray): +# """A type alias for raw data arrays used in the test workflow.""" + + def make_workflow(): + def sample_data_from_filename( + filename: Filename[SampleRun], + ) -> UnscaledReducibleData[SampleRun]: + return UnscaledReducibleData[SampleRun]( + make_sample_events(*(float(x) for x in filename.split('_'))) + ) + + def reference_data_from_filename( + filename: Filename[ReferenceRun], + ) -> UnscaledReducibleData[ReferenceRun]: + return UnscaledReducibleData[ReferenceRun]( + make_reference_events(*(float(x) for x in filename.split('_'))) + ) + def apply_scaling( da: UnscaledReducibleData[RunType], scale: ScalingFactorForOverlap[RunType], @@ -78,22 +96,27 @@ def reflectivity( ) -> ReflectivityOverQ: return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) - return sl.Pipeline([apply_scaling, reflectivity]) + return sl.Pipeline( + [ + sample_data_from_filename, + reference_data_from_filename, + apply_scaling, + reflectivity, + ] + ) def test_reflectivity_curve_scaling(): wf = make_workflow() wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - params = {'a': (1, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} + params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} for k, v in params.items(): - sample = make_sample_events(*v) - reference = make_reference_events(v[1], v[2]) workflows[k] = wf.copy() - workflows[k][UnscaledReducibleData[SampleRun]] = sample - workflows[k][UnscaledReducibleData[ReferenceRun]] = reference - workflows[k][QBins] = sample.coords['Q'] + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] wfc = WorkflowCollection(workflows) @@ -113,12 +136,10 @@ def test_reflectivity_curve_scaling_with_critical_edge(): params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} workflows = {} for k, v in params.items(): - sample = make_sample_events(*v) - reference = make_reference_events(v[1], v[2]) workflows[k] = wf.copy() - workflows[k][UnscaledReducibleData[SampleRun]] = sample - workflows[k][UnscaledReducibleData[ReferenceRun]] = reference - workflows[k][QBins] = sample.coords['Q'] + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] wfc = WorkflowCollection(workflows) @@ -132,44 +153,92 @@ def test_reflectivity_curve_scaling_with_critical_edge(): assert np.isclose(factors['b'], 0.5 / 0.8) assert np.isclose(factors['c'], 0.25 / 0.1) - # data = sc.concat( - # ( - # sc.ones(dims=['Q'], shape=[10], with_variances=True), - # 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - # ), - # dim='Q', - # ) - # data.variances[:] = 0.1 - - # curves, factors = scale_reflectivity_curves_to_overlap( - # ( - # 2 * curve(data, 0, 0.3), - # curve(0.8 * data, 0.2, 0.7), - # curve(0.1 * data, 0.6, 1.0), - # ), - # critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)), - # ) - - # assert_allclose(curves[0].data, data, rtol=sc.scalar(1e-5)) - # assert_allclose(curves[1].data, 0.5 * data, rtol=sc.scalar(1e-5)) - # assert_allclose(curves[2].data, 0.25 * data, rtol=sc.scalar(1e-5)) - # np_assert_allclose((0.5, 0.5 / 0.8, 0.25 / 0.1), factors, 1e-4) + +def test_reflectivity_curve_scaling_caches_intermediate_results(): + sample_count = 0 + reference_count = 0 + + def sample_data_from_filename( + filename: Filename[SampleRun], + ) -> UnscaledReducibleData[SampleRun]: + nonlocal sample_count + sample_count += 1 + return UnscaledReducibleData[SampleRun]( + make_sample_events(*(float(x) for x in filename.split('_'))) + ) + + def reference_data_from_filename( + filename: Filename[ReferenceRun], + ) -> UnscaledReducibleData[ReferenceRun]: + nonlocal reference_count + reference_count += 1 + return UnscaledReducibleData[ReferenceRun]( + make_reference_events(*(float(x) for x in filename.split('_'))) + ) + + def apply_scaling( + da: UnscaledReducibleData[RunType], + scale: ScalingFactorForOverlap[RunType], + ) -> ReducibleData[RunType]: + """ + Scales the raw data by a given factor. + """ + return ReducibleData[RunType](da * scale) + + def reflectivity( + sample: ReducibleData[SampleRun], + reference: ReducibleData[ReferenceRun], + qbins: QBins, + ) -> ReflectivityOverQ: + return ReflectivityOverQ(sample.hist(Q=qbins) / reference.hist(Q=qbins)) + + wf = sl.Pipeline( + [ + sample_data_from_filename, + reference_data_from_filename, + apply_scaling, + reflectivity, + ] + ) + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} + workflows = {} + for k, v in params.items(): + workflows[k] = wf.copy() + workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) + workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) + workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] + + wfc = WorkflowCollection(workflows) + + scaled_wf = scale_reflectivity_curves_to_overlap( + wfc, cache_intermediate_results=False + ) + scaled_wf.compute(ReflectivityOverQ) + # We expect 6 counts: 3 for each of the 3 runs * 2 for computing ReflectivityOverQ + # inside the scaling function and one more time for the final computation just above + assert sample_count == 6 + assert reference_count == 6 + + sample_count = 0 + reference_count = 0 + + scaled_wf = scale_reflectivity_curves_to_overlap( + wfc, cache_intermediate_results=True + ) + scaled_wf.compute(ReflectivityOverQ) + # We expect 3 counts: 1 for each of the 3 runs * 1 for computing ReflectivityOverQ + assert sample_count == 3 + assert reference_count == 3 def test_combined_curves(): qgrid = sc.linspace('Q', 0, 1, 26) - data = sc.concat( - ( - sc.ones(dims=['Q'], shape=[10], with_variances=True), - 0.5 * sc.ones(dims=['Q'], shape=[15], with_variances=True), - ), - dim='Q', - ) - data.variances[:] = 0.1 curves = ( - curve(data, 0, 0.3), - curve(0.5 * data, 0.2, 0.7), - curve(0.25 * data, 0.6, 1.0), + make_sample_events(1.0, 0, 0.3).hist(), + 0.5 * make_sample_events(1.0, 0.2, 0.7).hist(), + 0.25 * make_sample_events(1.0, 0.6, 1.0).hist(), ) combined = combine_curves(curves, qgrid) @@ -231,6 +300,7 @@ def test_combined_curves(): 0.00625, 0.00625, ], + unit='counts', ), ) @@ -314,7 +384,7 @@ def test_linlogspace_bad_input(): @pytest.mark.filterwarnings("ignore:No suitable") -def test_from_measurements_tool_uses_expected_parameters_from_each_run(): +def test_batch_processor_tool_uses_expected_parameters_from_each_run(): def normalized_ioq(filename: Filename[SampleRun]) -> ReflectivityOverQ: return filename @@ -329,103 +399,31 @@ class Reduction: workflow = sl.Pipeline( [normalized_ioq, orso_dataset], params={Filename[SampleRun]: 'default'} ) - datasets = from_measurements( - workflow, - [{}, {Filename[SampleRun]: 'special'}], - target=OrsoDataset, - scale_to_overlap=False, - ) - assert len(datasets) == 2 - assert tuple(d.info.name for d in datasets) == ('default.orso', 'special.orso') - - -@pytest.mark.parametrize('targets', [(int,), (float, int)]) -@pytest.mark.parametrize( - 'params', [[{str: '1'}, {str: '2'}], {'a': {str: '1'}, 'b': {str: '2'}}] -) -def test_from_measurements_tool_returns_mapping_if_passed_mapping(params, targets): - def A(x: str) -> float: - return float(x) - - def B(x: str) -> int: - return int(x) - - workflow = sl.Pipeline([A, B]) - datasets = from_measurements( - workflow, - params, - target=targets, - ) - assert len(datasets) == len(params) - assert type(datasets) is type(params) + batch = batch_processor(workflow, {'a': {}, 'b': {Filename[SampleRun]: 'special'}}) -def test_from_measurements_tool_does_not_recompute_reflectivity(): - R = sc.DataArray( - sc.ones(dims=['Q'], shape=(50,), with_variances=True), - coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, - ).bin(Q=10) + results = batch.compute(OrsoDataset) + assert len(results) == 2 + assert results['a'].info.name == 'default.orso' + assert results['b'].info.name == 'special.orso' - times_evaluated = 0 - def reflectivity() -> ReflectivityOverQ: - nonlocal times_evaluated - times_evaluated += 1 - return ReflectivityOverQ(R) - - def reducible_data() -> ReducibleData[SampleRun]: - return 'Not important' - - pl = sl.Pipeline([reflectivity, reducible_data]) - - from_measurements( - pl, - [{}, {}], - target=(ReflectivityOverQ,), - scale_to_overlap=True, - ) - assert times_evaluated == 2 - - -def test_from_measurements_tool_applies_scaling_to_reflectivityoverq(): - R1 = sc.DataArray( - sc.ones(dims=['Q'], shape=(50,), with_variances=True), - coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, - ).bin(Q=10) - R2 = 0.5 * R1 - - def reducible_data() -> ReducibleData[SampleRun]: - return 'Not important' - - pl = sl.Pipeline([reducible_data]) - - results = from_measurements( - pl, - [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], - target=(ReflectivityOverQ,), - scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), - ) - assert_allclose(results[0][ReflectivityOverQ], results[1][ReflectivityOverQ]) - - -def test_from_measurements_tool_applies_scaling_to_reducibledata(): - R1 = sc.DataArray( - sc.ones(dims=['Q'], shape=(50,), with_variances=True), - coords={'Q': sc.linspace('Q', 0.1, 1, 50)}, - ).bin(Q=10) - R2 = 0.5 * R1 +def test_batch_processor_tool_merges_event_lists(): + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - def reducible_data() -> ReducibleData[SampleRun]: - return sc.scalar(1) + runs = { + 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, + 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, + 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, + } + batch = batch_processor(wf, runs) - pl = sl.Pipeline([reducible_data]) + results = batch.compute(UnscaledReducibleData[SampleRun]) - results = from_measurements( - pl, - [{ReflectivityOverQ: R1}, {ReflectivityOverQ: R2}], - target=(ReducibleData[SampleRun],), - scale_to_overlap=(sc.scalar(0.0), sc.scalar(1.0)), - ) - assert_allclose( - results[0][ReducibleData[SampleRun]], 0.5 * results[1][ReducibleData[SampleRun]] + assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) + assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) + assert_almost_equal( + results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 ) From a8d1b3a014a0d575093f3df3789eedd2cb716254 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:43:24 +0200 Subject: [PATCH 22/29] fix scaling for a single workflow and fix amor pipeline tests --- src/ess/reflectometry/tools.py | 133 ++++++++++++++++-------------- tests/amor/pipeline_test.py | 13 +-- tests/amor/tools_test.py | 68 --------------- tests/reflectometry/tools_test.py | 17 ++++ 4 files changed, 96 insertions(+), 135 deletions(-) delete mode 100644 tests/amor/tools_test.py diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 587686bc..d15d7e6a 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -6,7 +6,7 @@ from typing import Any import numpy as np -import sciline +import sciline as sl import scipp as sc import scipy.optimize as opt @@ -109,6 +109,60 @@ def linlogspace( return sc.concat(grids, dim) +class WorkflowCollection: + """ + A collection of sciline workflows that can be used to compute multiple + targets from multiple workflows. + It can also be used to set parameters for all workflows in a single shot. + """ + + def __init__(self, workflows: Mapping[str, sl.Pipeline]): + self._workflows = {name: pl.copy() for name, pl in workflows.items()} + + def __setitem__(self, key: type, value: Any | Mapping[type, Any]): + if hasattr(value, 'items'): + for name, v in value.items(): + self._workflows[name][key] = v + else: + for pl in self._workflows.values(): + pl[key] = value + + def __getitem__(self, name: str) -> sl.Pipeline: + """ + Returns a single workflow from the collection given by its name. + """ + return self._workflows[name] + + def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + return { + name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() + } + + def copy(self) -> 'WorkflowCollection': + return self.__class__(self._workflows) + + def keys(self) -> Sequence[str]: + return self._workflows.keys() + + def values(self) -> Sequence[sl.Pipeline]: + return self._workflows.values() + + def items(self) -> Sequence[tuple[str, sl.Pipeline]]: + return self._workflows.items() + + def add(self, name: str, workflow: sl.Pipeline): + """ + Adds a new workflow to the collection. + """ + self._workflows[name] = workflow.copy() + + def remove(self, name: str): + """ + Removes a workflow from the collection by its name. + """ + del self._workflows[name] + + def _sort_by(a, by): return [x for x, _ in sorted(zip(a, by, strict=True), key=lambda x: x[1])] @@ -169,13 +223,14 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - wf_collection: Sequence[sc.DataArray], + workflows: WorkflowCollection | sl.Pipeline, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: ''' Set the ``ScalingFactorForOverlap`` parameter on the provided workflows in a way that would makes the 1D reflectivity curves overlap. + One can supply either a collection of workflows or a single workflow. If :code:`critical_edge_interval` is not provided, all workflows are scaled except the data with the lowest Q-range, which is considered to be the reference curve. @@ -188,8 +243,8 @@ def scale_reflectivity_curves_to_overlap( Parameters --------- - wf_collection: - The collection of workflows that can compute the ``ReflectivityOverQ``. + workflows: + The workflow or collection of workflows that can compute ``ReflectivityOverQ``. critical_edge_interval: A tuple denoting an interval that is known to belong to the critical edge, i.e. where the reflectivity is @@ -203,7 +258,17 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - wfc = wf_collection.copy() + if isinstance(workflows, sl.Pipeline): + # If a single workflow is provided, convert it to a collection + wfc = WorkflowCollection({"": workflows}) + out = scale_reflectivity_curves_to_overlap( + wfc, + critical_edge_interval=critical_edge_interval, + cache_intermediate_results=cache_intermediate_results, + ) + return out[""] + + wfc = workflows.copy() if cache_intermediate_results: wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( UnscaledReducibleData[SampleRun] @@ -226,7 +291,7 @@ def scale_reflectivity_curves_to_overlap( if critical_edge_interval is not None: # Find q bins with the lowest Q start point q = min( - (wf.compute(QBins) for wf in wf_collection.values()), + (wf.compute(QBins) for wf in workflows.values()), key=lambda q_: q_.min(), ) N = ( @@ -326,62 +391,8 @@ def combine_curves( ) -class WorkflowCollection: - """ - A collection of sciline workflows that can be used to compute multiple - targets from multiple workflows. - It can also be used to set parameters for all workflows in a single shot. - """ - - def __init__(self, workflows: Mapping[str, sciline.Pipeline]): - self._workflows = {name: pl.copy() for name, pl in workflows.items()} - - def __setitem__(self, key: type, value: Any | Mapping[type, Any]): - if hasattr(value, 'items'): - for name, v in value.items(): - self._workflows[name][key] = v - else: - for pl in self._workflows.values(): - pl[key] = value - - def __getitem__(self, name: str) -> sciline.Pipeline: - """ - Returns a single workflow from the collection given by its name. - """ - return self._workflows[name] - - def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - return { - name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() - } - - def copy(self) -> 'WorkflowCollection': - return self.__class__(self._workflows) - - def keys(self) -> Sequence[str]: - return self._workflows.keys() - - def values(self) -> Sequence[sciline.Pipeline]: - return self._workflows.values() - - def items(self) -> Sequence[tuple[str, sciline.Pipeline]]: - return self._workflows.items() - - def add(self, name: str, workflow: sciline.Pipeline): - """ - Adds a new workflow to the collection. - """ - self._workflows[name] = workflow.copy() - - def remove(self, name: str): - """ - Removes a workflow from the collection by its name. - """ - del self._workflows[name] - - def batch_processor( - workflow: sciline.Pipeline, runs: Mapping[Any, Mapping[type, Any]] + workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] ) -> WorkflowCollection: """ Creates a collection of sciline workflows from the provided runs. diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index b6ca1504..c03e4774 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -127,16 +127,17 @@ def test_save_reduced_orso_file(output_folder: Path): ) wf[Filename[ReferenceRun]] = data.amor_run(4152) wf[QBins] = sc.geomspace(dim="Q", start=0.01, stop=0.06, num=201, unit="1/angstrom") - r = wf.compute(ReflectivityOverQ) - _, (s,) = scale_reflectivity_curves_to_overlap( - [r.hist()], + # r = wf.compute(ReflectivityOverQ) + + scaled_wf = scale_reflectivity_curves_to_overlap( + wf, critical_edge_interval=( sc.scalar(0.01, unit='1/angstrom'), sc.scalar(0.014, unit='1/angstrom'), ), ) - wf[ReflectivityOverQ] = s * r - wf[orso.OrsoCreator] = orso.OrsoCreator( + # wf[ReflectivityOverQ] = s * r + scaled_wf[orso.OrsoCreator] = orso.OrsoCreator( fileio.base.Person( name="Max Mustermann", affiliation="European Spallation Source ERIC", @@ -144,7 +145,7 @@ def test_save_reduced_orso_file(output_folder: Path): ) ) fileio.orso.save_orso( - datasets=[wf.compute(orso.OrsoIofQDataset)], + datasets=[scaled_wf.compute(orso.OrsoIofQDataset)], fname=output_folder / 'amor_reduced_iofq.ort', ) diff --git a/tests/amor/tools_test.py b/tests/amor/tools_test.py deleted file mode 100644 index 0dd6fef2..00000000 --- a/tests/amor/tools_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -import pytest -import sciline -import scipp as sc -from scipp.testing import assert_allclose - -from amor.pipeline_test import amor_pipeline # noqa: F401 -from ess.amor import data -from ess.amor.types import ChopperPhase -from ess.reflectometry.tools import from_measurements -from ess.reflectometry.types import ( - DetectorRotation, - Filename, - QBins, - ReducedReference, - ReferenceRun, - ReflectivityOverQ, - SampleRotation, - SampleRun, -) - -# The files used in the AMOR reduction workflow have some scippnexus warnings -pytestmark = pytest.mark.filterwarnings( - "ignore:.*Invalid transformation, .*missing attribute 'vector':UserWarning", -) - - -@pytest.fixture -def pipeline_with_1632_reference(amor_pipeline): # noqa: F811 - amor_pipeline[ChopperPhase[ReferenceRun]] = sc.scalar(7.5, unit='deg') - amor_pipeline[ChopperPhase[SampleRun]] = sc.scalar(7.5, unit='deg') - amor_pipeline[Filename[ReferenceRun]] = data.amor_run('1632') - amor_pipeline[ReducedReference] = amor_pipeline.compute(ReducedReference) - return amor_pipeline - - -@pytestmark -def test_from_measurements_tool_concatenates_event_lists( - pipeline_with_1632_reference: sciline.Pipeline, -): - pl = pipeline_with_1632_reference - - run = { - Filename[SampleRun]: list(map(data.amor_run, (1636, 1639, 1641))), - QBins: sc.geomspace( - dim='Q', start=0.062, stop=0.18, num=391, unit='1/angstrom' - ), - DetectorRotation[SampleRun]: sc.scalar(0.140167, unit='rad'), - SampleRotation[SampleRun]: sc.scalar(0.0680678, unit='rad'), - } - results = from_measurements( - pl, - [run], - target=ReflectivityOverQ, - scale_to_overlap=False, - ) - - results2 = [] - for fname in run[Filename[SampleRun]]: - pl.copy() - pl[Filename[SampleRun]] = fname - pl[QBins] = run[QBins] - pl[DetectorRotation[SampleRun]] = run[DetectorRotation[SampleRun]] - pl[SampleRotation[SampleRun]] = run[SampleRotation[SampleRun]] - results2.append(pl.compute(ReflectivityOverQ).hist().data) - - assert_allclose(sum(results2), results[0].hist().data) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 4e089684..4b19524c 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -154,6 +154,23 @@ def test_reflectivity_curve_scaling_with_critical_edge(): assert np.isclose(factors['c'], 0.25 / 0.1) +def test_reflectivity_curve_scaling_works_with_single_workflow_and_critical_edge(): + wf = make_workflow() + wf[ScalingFactorForOverlap[SampleRun]] = 1.0 + wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 + wf[Filename[SampleRun]] = '2.5_0.4_0.8' + wf[Filename[ReferenceRun]] = '0.4_0.8' + wf[QBins] = make_reference_events(0.4, 0.8).coords['Q'] + + scaled_wf = scale_reflectivity_curves_to_overlap( + wf, critical_edge_interval=(sc.scalar(0.0), sc.scalar(0.5)) + ) + + factor = scaled_wf.compute(ScalingFactorForOverlap[SampleRun]) + + assert np.isclose(factor, 0.4) + + def test_reflectivity_curve_scaling_caches_intermediate_results(): sample_count = 0 reference_count = 0 From 75675654d1224d45039cf1a46d589312727d2ec5 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:49:04 +0200 Subject: [PATCH 23/29] cleanup --- src/ess/reflectometry/tools.py | 1 - tests/amor/pipeline_test.py | 2 -- tests/reflectometry/tools_test.py | 4 ---- 3 files changed, 7 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index d15d7e6a..18d7ac02 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -10,7 +10,6 @@ import scipp as sc import scipy.optimize as opt -# from ess.reflectometry import orso from ess.reflectometry.types import ( Filename, QBins, diff --git a/tests/amor/pipeline_test.py b/tests/amor/pipeline_test.py index c03e4774..9b5f01b3 100644 --- a/tests/amor/pipeline_test.py +++ b/tests/amor/pipeline_test.py @@ -127,7 +127,6 @@ def test_save_reduced_orso_file(output_folder: Path): ) wf[Filename[ReferenceRun]] = data.amor_run(4152) wf[QBins] = sc.geomspace(dim="Q", start=0.01, stop=0.06, num=201, unit="1/angstrom") - # r = wf.compute(ReflectivityOverQ) scaled_wf = scale_reflectivity_curves_to_overlap( wf, @@ -136,7 +135,6 @@ def test_save_reduced_orso_file(output_folder: Path): sc.scalar(0.014, unit='1/angstrom'), ), ) - # wf[ReflectivityOverQ] = s * r scaled_wf[orso.OrsoCreator] = orso.OrsoCreator( fileio.base.Person( name="Max Mustermann", diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 4b19524c..72e108df 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -61,10 +61,6 @@ def make_reference_events(qmin, qmax): return data.bin(Q=qbins) -# class RawData(sl.Scope[RunType, sc.DataArray], sc.DataArray): -# """A type alias for raw data arrays used in the test workflow.""" - - def make_workflow(): def sample_data_from_filename( filename: Filename[SampleRun], From 7f884fb3fe1db11895d2edd647fb3f0beab6b4d3 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 7 Aug 2025 14:55:59 +0200 Subject: [PATCH 24/29] do not fail if UnscaledReducibleData is not in graph --- src/ess/reflectometry/tools.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 18d7ac02..434200aa 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -269,12 +269,18 @@ def scale_reflectivity_curves_to_overlap( wfc = workflows.copy() if cache_intermediate_results: - wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( - UnscaledReducibleData[SampleRun] - ) - wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( - UnscaledReducibleData[ReferenceRun] - ) + try: + wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( + UnscaledReducibleData[SampleRun] + ) + except sl.UnsatisfiedRequirement: + pass + try: + wfc[UnscaledReducibleData[ReferenceRun]] = wfc.compute( + UnscaledReducibleData[ReferenceRun] + ) + except sl.UnsatisfiedRequirement: + pass reflectivities = wfc.compute(ReflectivityOverQ) From 2dea8de9bc081e6a92192c6f95e757e3fc8ebd9f Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 13 Aug 2025 16:06:35 +0200 Subject: [PATCH 25/29] modify the WorkflowCollection to just use cyclebane mapping under the hood and make its interface more like the Pipeline interface rather than a mix between Pipeline and a dict --- src/ess/reflectometry/tools.py | 168 ++++++++++++++---------------- tests/reflectometry/tools_test.py | 86 +++++++-------- 2 files changed, 120 insertions(+), 134 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 434200aa..a5416484 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -11,7 +11,6 @@ import scipy.optimize as opt from ess.reflectometry.types import ( - Filename, QBins, ReferenceRun, ReflectivityOverQ, @@ -19,7 +18,6 @@ ScalingFactorForOverlap, UnscaledReducibleData, ) -from ess.reflectometry.workflow import with_filenames _STD_TO_FWHM = sc.scalar(2.0) * sc.sqrt(sc.scalar(2.0) * sc.log(sc.scalar(2.0))) @@ -111,55 +109,50 @@ def linlogspace( class WorkflowCollection: """ A collection of sciline workflows that can be used to compute multiple - targets from multiple workflows. - It can also be used to set parameters for all workflows in a single shot. + targets from mapping a workflow over a parameter table. + It can also be used to set parameters for all mapped nodes in a single shot. """ - def __init__(self, workflows: Mapping[str, sl.Pipeline]): - self._workflows = {name: pl.copy() for name, pl in workflows.items()} + def __init__(self, workflow: sl.Pipeline, param_table): + self._original_workflow = workflow.copy() + self.param_table = param_table + self._mapped_workflow = self._original_workflow.map(self.param_table) - def __setitem__(self, key: type, value: Any | Mapping[type, Any]): - if hasattr(value, 'items'): - for name, v in value.items(): - self._workflows[name][key] = v + def __setitem__(self, key, value): + if key in self.param_table: + ind = list(self.param_table.keys()).index(key) + self.param_table.iloc[:, ind] = value + self._mapped_workflow = self._original_workflow.map(self.param_table) else: - for pl in self._workflows.values(): - pl[key] = value - - def __getitem__(self, name: str) -> sl.Pipeline: - """ - Returns a single workflow from the collection given by its name. - """ - return self._workflows[name] + self.param_table.insert(len(self.param_table.columns), key, value) + self._original_workflow[key] = None + self._mapped_workflow = self._original_workflow.map(self.param_table) def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - return { - name: pl.compute(target, **kwargs) for name, pl in self._workflows.items() - } - - def copy(self) -> 'WorkflowCollection': - return self.__class__(self._workflows) - - def keys(self) -> Sequence[str]: - return self._workflows.keys() + try: + # TODO: Sciline here returns a pandas Series. + # Should we convert it to a dict instead? + return sl.compute_mapped(self._mapped_workflow, target, **kwargs) + except ValueError: + return self._mapped_workflow.compute(target, **kwargs) - def values(self) -> Sequence[sl.Pipeline]: - return self._workflows.values() + def get(self, targets, **kwargs): + try: + targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + return self._mapped_workflow.get(targets, **kwargs) + except ValueError: + return self._mapped_workflow.get(targets, **kwargs) - def items(self) -> Sequence[tuple[str, sl.Pipeline]]: - return self._workflows.items() + # TODO: implement the group() method to group by params in the parameter table - def add(self, name: str, workflow: sl.Pipeline): - """ - Adds a new workflow to the collection. - """ - self._workflows[name] = workflow.copy() + def visualize(self, targets, **kwargs): + targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + return self._mapped_workflow.visualize(targets, **kwargs) - def remove(self, name: str): - """ - Removes a workflow from the collection by its name. - """ - del self._workflows[name] + def copy(self) -> 'WorkflowCollection': + return self.__class__( + workflow=self._original_workflow, param_table=self.param_table + ) def _sort_by(a, by): @@ -222,7 +215,7 @@ def _interpolate_on_qgrid(curves, grid): def scale_reflectivity_curves_to_overlap( - workflows: WorkflowCollection | sl.Pipeline, + workflow: WorkflowCollection | sl.Pipeline, critical_edge_interval: tuple[sc.Variable, sc.Variable] | None = None, cache_intermediate_results: bool = True, ) -> tuple[list[sc.DataArray], list[sc.Variable]]: @@ -257,17 +250,9 @@ def scale_reflectivity_curves_to_overlap( : A list of scaled reflectivity curves and a list of the scaling factors. ''' - if isinstance(workflows, sl.Pipeline): - # If a single workflow is provided, convert it to a collection - wfc = WorkflowCollection({"": workflows}) - out = scale_reflectivity_curves_to_overlap( - wfc, - critical_edge_interval=critical_edge_interval, - cache_intermediate_results=cache_intermediate_results, - ) - return out[""] + not_collection = isinstance(workflow, sl.Pipeline) - wfc = workflows.copy() + wfc = workflow.copy() if cache_intermediate_results: try: wfc[UnscaledReducibleData[SampleRun]] = wfc.compute( @@ -283,6 +268,8 @@ def scale_reflectivity_curves_to_overlap( pass reflectivities = wfc.compute(ReflectivityOverQ) + if not_collection: + reflectivities = {"": reflectivities} # First sort the dict of reflectivities by the Q min value curves = { @@ -294,20 +281,21 @@ def scale_reflectivity_curves_to_overlap( critical_edge_key = uuid.uuid4().hex if critical_edge_interval is not None: - # Find q bins with the lowest Q start point - q = min( - (wf.compute(QBins) for wf in workflows.values()), - key=lambda q_: q_.min(), - ) - N = ( - ((q >= critical_edge_interval[0]) & (q < critical_edge_interval[1])) - .sum() - .value - ) + q = wfc.compute(QBins) + if hasattr(q, "items"): + # If QBins is a mapping, find the one with the lowest Q start + # Note the conversion to a dict, because if pandas is used for the mapping, + # it will return a Series, whose `.values` attribute is not callable. + q = min(dict(q).values(), key=lambda q_: q_.min()) + + # TODO: This is slightly different from before: it extracts the bins from the + # QBins variable that cover the critical edge interval. This means that the + # resulting curve will not necessarily begin and end exactly at the values + # specified, but rather at the closest bin edges. edge = sc.DataArray( - data=sc.ones(dims=('Q',), shape=(N,), with_variances=True), - coords={'Q': sc.linspace('Q', *critical_edge_interval, N + 1)}, - ) + data=sc.ones(sizes={q.dim: q.sizes[q.dim] - 1}, with_variances=True), + coords={q.dim: q}, + )[q.dim, critical_edge_interval[0] : critical_edge_interval[1]] # Now place the critical edge at the beginning curves = {critical_edge_key: edge} | curves @@ -335,11 +323,14 @@ def cost(scaling_factors): sol = opt.minimize(cost, [1.0] * (len(curves) - 1)) scaling_factors = (1.0, *map(float, sol.x)) - wfc[ScalingFactorForOverlap[SampleRun]] = { + results = { k: v for k, v in zip(curves.keys(), scaling_factors, strict=True) if k != critical_edge_key } + if not_collection: + results = results[""] + wfc[ScalingFactorForOverlap[SampleRun]] = results return wfc @@ -397,10 +388,10 @@ def combine_curves( def batch_processor( - workflow: sl.Pipeline, runs: Mapping[Any, Mapping[type, Any]] + workflow: sl.Pipeline, params: Mapping[Any, Mapping[type, Any]] ) -> WorkflowCollection: """ - Creates a collection of sciline workflows from the provided runs. + Maps the provided workflow over the provided params. Example: @@ -415,7 +406,7 @@ def batch_processor( Filename[SampleRun]: amor.data.amor_run(608), }, '609': { - SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + SampleRotationOffset[SampleRun]: sc.scalar(0.06, unit='deg'), Filename[SampleRun]: amor.data.amor_run(609), }, '610': { @@ -423,7 +414,7 @@ def batch_processor( Filename[SampleRun]: amor.data.amor_run(610), }, '611': { - SampleRotationOffset[SampleRun]: sc.scalar(0.05, unit='deg'), + SampleRotationOffset[SampleRun]: sc.scalar(0.07, unit='deg'), Filename[SampleRun]: amor.data.amor_run(611), }, } @@ -437,30 +428,23 @@ def batch_processor( ---------- workflow: The sciline workflow used to compute the targets for each of the runs. - runs: + params: The sciline parameters to be used for each run. Should be a mapping where the keys are the names of the runs and the values are mappings of type to value pairs. - In addition, if one of the values for ``Filename[SampleRun]`` - is a list or a tuple, then the events from the files - will be concatenated into a single event list. """ - workflows = {} - for name, parameters in runs.items(): - wf = workflow.copy() - for tp, value in parameters.items(): - if tp is Filename[SampleRun]: - continue - wf[tp] = value - - if Filename[SampleRun] in parameters: - if isinstance(parameters[Filename[SampleRun]], list | tuple): - wf = with_filenames( - wf, - SampleRun, - parameters[Filename[SampleRun]], - ) + import pandas as pd + + all_types = {t for v in params.values() for t in v.keys()} + data = {t: [] for t in all_types} + for param in params.values(): + for t in all_types: + if t in param: + data[t].append(param[t]) else: - wf[Filename[SampleRun]] = parameters[Filename[SampleRun]] - workflows[name] = wf - return WorkflowCollection(workflows) + # Set the default value + data[t].append(workflow.compute(t)) + + param_table = pd.DataFrame(data, index=params.keys()).rename_axis(index='run_id') + + return WorkflowCollection(workflow, param_table) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 72e108df..256b3d76 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -5,12 +5,10 @@ import pytest import sciline as sl import scipp as sc -from numpy.testing import assert_almost_equal from orsopy.fileio import Orso, OrsoDataset from scipp.testing import assert_allclose from ess.reflectometry.tools import ( - WorkflowCollection, batch_processor, combine_curves, linlogspace, @@ -107,14 +105,15 @@ def test_reflectivity_curve_scaling(): wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - workflows = {} - for k, v in params.items(): - workflows[k] = wf.copy() - workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) - workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) - workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - - wfc = WorkflowCollection(workflows) + table = { + k: { + Filename[SampleRun]: "_".join(map(str, v)), + Filename[ReferenceRun]: "_".join(map(str, v[1:])), + QBins: make_reference_events(*v[1:]).coords['Q'], + } + for k, v in params.items() + } + wfc = batch_processor(wf, table) scaled_wf = scale_reflectivity_curves_to_overlap(wfc) @@ -130,14 +129,15 @@ def test_reflectivity_curve_scaling_with_critical_edge(): wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (2, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - workflows = {} - for k, v in params.items(): - workflows[k] = wf.copy() - workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) - workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) - workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - - wfc = WorkflowCollection(workflows) + table = { + k: { + Filename[SampleRun]: "_".join(map(str, v)), + Filename[ReferenceRun]: "_".join(map(str, v[1:])), + QBins: make_reference_events(*v[1:]).coords['Q'], + } + for k, v in params.items() + } + wfc = batch_processor(wf, table) scaled_wf = scale_reflectivity_curves_to_overlap( wfc, critical_edge_interval=(sc.scalar(0.01), sc.scalar(0.05)) @@ -216,14 +216,15 @@ def reflectivity( wf[ScalingFactorForOverlap[SampleRun]] = 1.0 wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 params = {'a': (1.0, 0, 0.3), 'b': (0.8, 0.2, 0.7), 'c': (0.1, 0.6, 1.0)} - workflows = {} - for k, v in params.items(): - workflows[k] = wf.copy() - workflows[k][Filename[SampleRun]] = "_".join(map(str, v)) - workflows[k][Filename[ReferenceRun]] = "_".join(map(str, v[1:])) - workflows[k][QBins] = make_reference_events(*v[1:]).coords['Q'] - - wfc = WorkflowCollection(workflows) + table = { + k: { + Filename[SampleRun]: "_".join(map(str, v)), + Filename[ReferenceRun]: "_".join(map(str, v[1:])), + QBins: make_reference_events(*v[1:]).coords['Q'], + } + for k, v in params.items() + } + wfc = batch_processor(wf, table) scaled_wf = scale_reflectivity_curves_to_overlap( wfc, cache_intermediate_results=False @@ -421,22 +422,23 @@ class Reduction: assert results['b'].info.name == 'special.orso' -def test_batch_processor_tool_merges_event_lists(): - wf = make_workflow() - wf[ScalingFactorForOverlap[SampleRun]] = 1.0 - wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 +# TODO: need to implement groupby in the mapping +# def test_batch_processor_tool_merges_event_lists(): +# wf = make_workflow() +# wf[ScalingFactorForOverlap[SampleRun]] = 1.0 +# wf[ScalingFactorForOverlap[ReferenceRun]] = 1.0 - runs = { - 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, - 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, - 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, - } - batch = batch_processor(wf, runs) +# runs = { +# 'a': {Filename[SampleRun]: ('1.0_0.0_0.3', '1.5_0.0_0.3')}, +# 'b': {Filename[SampleRun]: '0.8_0.2_0.7'}, +# 'c': {Filename[SampleRun]: ('0.1_0.6_1.0', '0.2_0.6_1.0')}, +# } +# batch = batch_processor(wf, runs) - results = batch.compute(UnscaledReducibleData[SampleRun]) +# results = batch.compute(UnscaledReducibleData[SampleRun]) - assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) - assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) - assert_almost_equal( - results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 - ) +# assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) +# assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) +# assert_almost_equal( +# results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 +# ) From f181b41d42bee5f633c3eec5b8802da70f2c2282 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 13 Aug 2025 16:07:18 +0200 Subject: [PATCH 26/29] formatting --- tests/reflectometry/tools_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/reflectometry/tools_test.py b/tests/reflectometry/tools_test.py index 256b3d76..70e8838e 100644 --- a/tests/reflectometry/tools_test.py +++ b/tests/reflectometry/tools_test.py @@ -437,7 +437,8 @@ class Reduction: # results = batch.compute(UnscaledReducibleData[SampleRun]) -# assert_almost_equal(results['a'].sum().value, 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) +# assert_almost_equal(results['a'].sum().value, +# 10 + 15 * 0.5 + (10 + 15 * 0.5) * 1.5) # assert_almost_equal(results['b'].sum().value, 10 * 0.8 + 15 * 0.5 * 0.8) # assert_almost_equal( # results['c'].sum().value, (10 + 15 * 0.5) * 0.1 + (10 + 15 * 0.5) * 0.2 From a23c007e51646b0f63bd8a6e61ea6003c168238e Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Wed, 13 Aug 2025 16:10:42 +0200 Subject: [PATCH 27/29] update amor notebook --- docs/user-guide/amor/amor-reduction.ipynb | 27 ++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/user-guide/amor/amor-reduction.ipynb b/docs/user-guide/amor/amor-reduction.ipynb index 18d9224e..c0c08ec0 100644 --- a/docs/user-guide/amor/amor-reduction.ipynb +++ b/docs/user-guide/amor/amor-reduction.ipynb @@ -184,8 +184,15 @@ "}\n", "\n", "batch = batch_processor(workflow, runs)\n", - "display(batch.keys())\n", - "\n", + "batch.param_table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "# Compute R(Q) for all runs\n", "reflectivity = batch.compute(ReflectivityOverQ)\n", "sc.plot(\n", @@ -288,8 +295,8 @@ "from ess.reflectometry.figures import wavelength_theta_figure\n", "\n", "wavelength_theta_figure(\n", - " diagnostics.values(),\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", + " diagnostics.values,\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values,\n", " q_edges_to_display=(sc.scalar(0.018, unit='1/angstrom'), sc.scalar(0.113, unit='1/angstrom'))\n", ")" ] @@ -318,8 +325,8 @@ "from ess.reflectometry.figures import q_theta_figure\n", "\n", "q_theta_figure(\n", - " diagnostics.values(),\n", - " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values(),\n", + " diagnostics.values,\n", + " theta_bins=scaled_wf.compute(ThetaBins[SampleRun]).values,\n", " q_bins=workflow.compute(QBins)\n", ")" ] @@ -396,7 +403,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can visualize the workflow for a single run (`'608'`):" + "We can visualize the workflow for the `OrsoIofQDataset`:" ] }, { @@ -405,7 +412,7 @@ "metadata": {}, "outputs": [], "source": [ - "scaled_wf['608'].visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" + "scaled_wf.visualize(orso.OrsoIofQDataset, graph_attr={'rankdir': 'LR'})" ] }, { @@ -437,7 +444,7 @@ "metadata": {}, "outputs": [], "source": [ - "for ds in iofq_datasets.values():\n", + "for ds in iofq_datasets.values:\n", " ds.info.reduction.script = (\n", " 'https://scipp.github.io/essreflectometry/user-guide/amor/amor-reduction.html'\n", " )" @@ -457,7 +464,7 @@ "metadata": {}, "outputs": [], "source": [ - "fileio.orso.save_orso(datasets=list(iofq_datasets.values()), fname='amor_reduced_iofq.ort')" + "fileio.orso.save_orso(datasets=iofq_datasets.values, fname='amor_reduced_iofq.ort')" ] }, { From ccfb1b74f50ebc4de73504eed6b0755bf027c577 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Fri, 15 Aug 2025 11:12:12 +0200 Subject: [PATCH 28/29] debugging compute multiple --- src/ess/reflectometry/tools.py | 87 ++++++++++++++++--- .../reflectometry/workflow_collection_test.py | 34 +++++--- 2 files changed, 95 insertions(+), 26 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index a5416484..247fdd7d 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -128,20 +128,79 @@ def __setitem__(self, key, value): self._original_workflow[key] = None self._mapped_workflow = self._original_workflow.map(self.param_table) - def compute(self, target: type | Sequence[type], **kwargs) -> Mapping[str, Any]: - try: - # TODO: Sciline here returns a pandas Series. - # Should we convert it to a dict instead? - return sl.compute_mapped(self._mapped_workflow, target, **kwargs) - except ValueError: - return self._mapped_workflow.compute(target, **kwargs) - - def get(self, targets, **kwargs): - try: - targets = sl.get_mapped_node_names(self._mapped_workflow, targets) - return self._mapped_workflow.get(targets, **kwargs) - except ValueError: - return self._mapped_workflow.get(targets, **kwargs) + def compute(self, keys: type | Sequence[type], **kwargs) -> Mapping[str, Any]: + from sciline.pipeline import _is_multiple_keys + + out = {} + if _is_multiple_keys(keys): + for key in keys: + if sl.is_mapped_node(self._mapped_workflow, key): + targets = [ + n + for x in key + for n in sl.get_mapped_node_names(self._mapped_workflow, x) + ] + results = self._mapped_workflow.compute(targets, **kwargs) + for node, v in results.items(): + key = node.index.values[0] + if key not in out: + out[key] = {node.name: [v]} + else: + out[key][node.name] = v + + # if sl.is_mapped_node(target): + # from sciline.pipeline import _is_multiple_keys + + # targets = [ + # n + # for x in target + # for n in sl.get_mapped_node_names(self._mapped_workflow, x) + # ] + # results = self._mapped_workflow.compute(targets, **kwargs) + # out = {} + # for node, v in results.items(): + # key = node.index.values[0] + # if key not in out: + # out[key] = {node.name: [v]} + # else: + # out[key][node.name] = v + # return out + # else: + # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) + # else: + # return self._mapped_workflow.compute(target, **kwargs) + + # def get(self, keys, **kwargs): + # if _is_multiple_keys(target): + + # if sl.is_mapped_node(target): + # from sciline.pipeline import _is_multiple_keys + + # if _is_multiple_keys(target): + # targets = [ + # n + # for x in target + # for n in sl.get_mapped_node_names(self._mapped_workflow, x) + # ] + + # results = self._mapped_workflow.compute(targets, **kwargs) + # out = {} + # for node, v in results.items(): + # key = node.index.values[0] + # if key not in out: + # out[key] = {node.name: [v]} + # else: + # out[key][node.name] = v + # return out + # else: + # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) + # else: + # return self._mapped_workflow.compute(target, **kwargs) + # # try: + # # targets = sl.get_mapped_node_names(self._mapped_workflow, targets) + # # return self._mapped_workflow.get(targets, **kwargs) + # # except ValueError: + # # return self._mapped_workflow.get(targets, **kwargs) # TODO: implement the group() method to group by params in the parameter table diff --git a/tests/reflectometry/workflow_collection_test.py b/tests/reflectometry/workflow_collection_test.py index bb1ff529..a7232a72 100644 --- a/tests/reflectometry/workflow_collection_test.py +++ b/tests/reflectometry/workflow_collection_test.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import pandas as pd import sciline as sl from ess.reflectometry.tools import WorkflowCollection @@ -14,25 +15,34 @@ def int_float_to_str(x: int, y: float) -> str: return f"{x};{y}" +def make_param_table(params: dict) -> pd.DataFrame: + all_types = {t for v in params.values() for t in v.keys()} + data = {t: [] for t in all_types} + for param in params.values(): + for t in all_types: + data[t].append(param[t]) + return pd.DataFrame(data, index=params.keys()).rename_axis(index='run_id') + + def test_compute() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) - assert coll.compute(float) == {'a': 1.5, 'b': 2.0} - assert coll.compute(str) == {'a': '3;1.5', 'b': '4;2.0'} + coll = WorkflowCollection(wf, make_param_table({'a': {int: 3}, 'b': {int: 4}})) + + assert dict(coll.compute(float)) == {'a': 1.5, 'b': 2.0} + assert dict(coll.compute(str)) == {'a': '3;1.5', 'b': '4;2.0'} def test_compute_multiple() -> None: wf = sl.Pipeline([int_to_float, int_float_to_str]) - wfa = wf.copy() - wfa[int] = 3 - wfb = wf.copy() - wfb[int] = 4 - coll = WorkflowCollection({'a': wfa, 'b': wfb}) + + coll = WorkflowCollection(wf, make_param_table({'a': {int: 3}, 'b': {int: 4}})) + + # wfa = wf.copy() + # wfa[int] = 3 + # wfb = wf.copy() + # wfb[int] = 4 + # coll = WorkflowCollection({'a': wfa, 'b': wfb}) result = coll.compute([float, str]) From c8e43a9e94df4015cfe95bdc4d10dc37ebc86a77 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Tue, 19 Aug 2025 12:25:07 +0200 Subject: [PATCH 29/29] fix compute --- src/ess/reflectometry/tools.py | 83 ++++++---------------------------- 1 file changed, 14 insertions(+), 69 deletions(-) diff --git a/src/ess/reflectometry/tools.py b/src/ess/reflectometry/tools.py index 247fdd7d..b0808e65 100644 --- a/src/ess/reflectometry/tools.py +++ b/src/ess/reflectometry/tools.py @@ -132,75 +132,20 @@ def compute(self, keys: type | Sequence[type], **kwargs) -> Mapping[str, Any]: from sciline.pipeline import _is_multiple_keys out = {} - if _is_multiple_keys(keys): - for key in keys: - if sl.is_mapped_node(self._mapped_workflow, key): - targets = [ - n - for x in key - for n in sl.get_mapped_node_names(self._mapped_workflow, x) - ] - results = self._mapped_workflow.compute(targets, **kwargs) - for node, v in results.items(): - key = node.index.values[0] - if key not in out: - out[key] = {node.name: [v]} - else: - out[key][node.name] = v - - # if sl.is_mapped_node(target): - # from sciline.pipeline import _is_multiple_keys - - # targets = [ - # n - # for x in target - # for n in sl.get_mapped_node_names(self._mapped_workflow, x) - # ] - # results = self._mapped_workflow.compute(targets, **kwargs) - # out = {} - # for node, v in results.items(): - # key = node.index.values[0] - # if key not in out: - # out[key] = {node.name: [v]} - # else: - # out[key][node.name] = v - # return out - # else: - # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) - # else: - # return self._mapped_workflow.compute(target, **kwargs) - - # def get(self, keys, **kwargs): - # if _is_multiple_keys(target): - - # if sl.is_mapped_node(target): - # from sciline.pipeline import _is_multiple_keys - - # if _is_multiple_keys(target): - # targets = [ - # n - # for x in target - # for n in sl.get_mapped_node_names(self._mapped_workflow, x) - # ] - - # results = self._mapped_workflow.compute(targets, **kwargs) - # out = {} - # for node, v in results.items(): - # key = node.index.values[0] - # if key not in out: - # out[key] = {node.name: [v]} - # else: - # out[key][node.name] = v - # return out - # else: - # return dict(sl.compute_mapped(self._mapped_workflow, target, **kwargs)) - # else: - # return self._mapped_workflow.compute(target, **kwargs) - # # try: - # # targets = sl.get_mapped_node_names(self._mapped_workflow, targets) - # # return self._mapped_workflow.get(targets, **kwargs) - # # except ValueError: - # # return self._mapped_workflow.get(targets, **kwargs) + if not _is_multiple_keys(keys): + keys = [keys] + for key in keys: + out[key] = {} + if sl.is_mapped_node(self._mapped_workflow, key): + targets = sl.get_mapped_node_names(self._mapped_workflow, key) + results = self._mapped_workflow.compute(targets, **kwargs) + for node, v in results.items(): + out[key][node.index.values[0]] = v + else: + out[key] = self._mapped_workflow.compute(key, **kwargs) + return next(iter(out.values())) if len(out) == 1 else out + + # TODO: implement get() # TODO: implement the group() method to group by params in the parameter table