From 3bcf88a6de32c66b3fef79ebebea9b5d4f8e05e8 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Thu, 15 Oct 2026 16:56:04 -0400 Subject: [PATCH 1/6] CI: test on Python 3.10, 3.12 and 3.13 (drop 3.11) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Xy8Dg7fAqocHGzMqVCiSK --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7cf9203..f0a4a06 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,7 @@ concurrency: cancel-in-progress: true env: - STABLE_PYTHON_VERSION: '3.11' + STABLE_PYTHON_VERSION: '3.12' PYTHONUNBUFFERED: "1" FORCE_COLOR: "1" @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.12", "3.13"] os: [ubuntu-latest, windows-latest, macos-latest] steps: From 47172c95404c0a502f84830247c913b95e2ef9e1 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Thu, 15 Oct 2026 16:56:13 -0400 Subject: [PATCH 2/6] Add seeding to build_vset and fix bugs found in review Seeding: build_vset(..., reps=n, seed=s) gives each vfunc its own random_state drawn from np.random.SeedSequence(s), so repeated perturbations (e.g. bootstrap samples) are reproducible. Bug fixes (each has a regression test in tests/test_bugfixes.py): - filter_vset_by_metric kept nothing for build_vset output without param_dict or with reps, double-counted configurations across upstream perturbations, and could not filter a filtered Vset - dict_to_df(param_key=...) mixed up parameters across rows and split values containing '=' - perturbation_stats: operator-precedence bug, string/callable func, prefix applied to default columns - PCSPipeline.run did not chain steps; generate_names crashed - MLflow tracking: params only logged in some cases, runs never closed, non-numeric outputs (fit/predict) crashed logging - caching served stale results after a function was redefined and failed on lambdas; cache now keyed on a fingerprint of the inputs - Vset accepted bad containers and duplicate keys, ignored async vfuncs given as a dict, and dict keys were not turned into Subkeys - predict/predict_proba on a Vset without a predictor now raise - lazy async outputs feeding a lazy sync Vset were not resolved - Subkey hash disagreed with equality - cum_acc_by_uncertainty gave an unclear error on class predictions Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Xy8Dg7fAqocHGzMqVCiSK --- README.md | 9 +- tests/test_bugfixes.py | 271 +++++++++++++++++++++++++++++++++++++++++ tests/test_seeding.py | 86 +++++++++++++ vflow/helpers.py | 120 +++++++++++++++--- vflow/pipeline.py | 31 +++-- vflow/subkey.py | 5 +- vflow/utils.py | 45 ++++--- vflow/vfunc.py | 6 + vflow/vset.py | 102 ++++++++++++++-- 9 files changed, 611 insertions(+), 64 deletions(-) create mode 100644 tests/test_bugfixes.py create mode 100644 tests/test_seeding.py diff --git a/README.md b/README.md index f9fc9ff..a8163b8 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ from sklearn.metrics import accuracy_score, balanced_accuracy_score from sklearn.model_selection import train_test_split from sklearn.tree import DecisionTreeClassifier -from vflow import Vset, init_args +from vflow import Vset, build_vset, init_args # initialize data X, y = make_classification() @@ -52,10 +52,9 @@ X_train, X_test, y_train, y_test = init_args( names=["X_train", "X_test", "y_train", "y_test"], # optionally name the args ) -# subsample data -subsampling_funcs = [sklearn.utils.resample for _ in range(3)] -subsampling_set = Vset( - name="subsampling", vfuncs=subsampling_funcs, output_matching=True +# subsample data: 3 bootstrap samples, reproducible thanks to the seed +subsampling_set = build_vset( + "subsampling", sklearn.utils.resample, reps=3, seed=0, output_matching=True ) X_trains, y_trains = subsampling_set(X_train, y_train) diff --git a/tests/test_bugfixes.py b/tests/test_bugfixes.py new file mode 100644 index 0000000..96cf223 --- /dev/null +++ b/tests/test_bugfixes.py @@ -0,0 +1,271 @@ +"""Regression tests for bugs found by review, one test per bug.""" + +import numpy as np +import pandas as pd +import pytest + +from vflow import ( + PCSPipeline, + Vset, + build_vset, + cum_acc_by_uncertainty, + dict_to_df, + filter_vset_by_metric, + init_args, + perturbation_stats, +) + + +def values(d): + """The outputs in a Vset output dict, without its bookkeeping entry.""" + return [v for k, v in d.items() if k != "__prev__"] + + +def model(X, a=1): + """A 'model' whose score is just its parameter, so the best one is known.""" + return float(a) + 0 * np.sum(X) + + +class TestFilterVsetByMetric: + def setup_method(self): + (self.X,) = init_args([np.ones((4, 2))], names=["X"]) + + def test_vsets_without_param_dict(self): + def low(X): + return 1.0 + + def high(X): + return 2.0 + + vset = build_vset("plain", [low, high]) + kept = filter_vset_by_metric(vset(self.X), vset, n_keep=1) + assert len(kept.vfuncs) == 1, "used to keep nothing" + assert next(iter(kept.vfuncs.values())).vfunc(None) == 2.0 + + def test_vsets_with_reps(self): + vset = build_vset("m", model, param_dict={"a": [1, 3]}, reps=2) + kept = filter_vset_by_metric(vset(self.X), vset, n_keep=1) + assert len(kept.vfuncs) == 1, "used to keep nothing when reps > 1" + + def test_n_keep_counts_distinct_vfuncs(self): + boot = Vset("boot", [lambda X: X, lambda X: 2 * X], vfunc_keys=["b0", "b1"]) + mset = build_vset("m", model, param_dict={"a": [1, 2, 3]}) + scores = mset(boot(self.X)) + kept = filter_vset_by_metric(scores, mset, n_keep=2) + # a=3 tops both bootstrap rows; it must count once + assert len(kept.vfuncs) == 2 + assert {k[0].value for k in kept.vfuncs} == { + ("func=model", "a=3"), + ("func=model", "a=2"), + } + + def test_filtering_a_filtered_vset(self): + boot = Vset("boot", [lambda X: X], vfunc_keys=["b0"]) + mset = build_vset("m", model, param_dict={"a": [1, 2, 3]}) + subs = boot(self.X) + once = filter_vset_by_metric(mset(subs), mset, n_keep=2) + twice = filter_vset_by_metric(once(subs), once, n_keep=1) + assert [k[0].value for k in twice.vfuncs] == [("func=model", "a=3")] + + +class TestPerturbationStats: + def setup_method(self): + self.df = pd.DataFrame({"g": ["a", "a", "b", "b"], "out": [1.0, 3.0, 2.0, 6.0]}) + + def test_callable_and_string_func(self): + by_mean = perturbation_stats(self.df, "g", func=np.mean) + assert list(by_mean["mean"]) == [2.0, 4.0] + by_std = perturbation_stats(self.df, "g", func="std") + assert list(by_std.columns) == ["g", "std"] + + def test_default_columns_unchanged_and_prefix_honoured(self): + assert list(perturbation_stats(self.df, "g").columns) == [ + "g", + "count", + "mean", + "std", + ] + cols = perturbation_stats(self.df, "g", prefix="acc").columns + assert list(cols) == ["g", "acc-count", "acc-mean", "acc-std"] + + def test_array_outputs_without_count(self): + df = pd.DataFrame( + {"g": ["a", "a"], "out": [np.array([1.0, 2.0]), np.array([3.0, 4.0])]} + ) + out = perturbation_stats(df, "g", func=["mean"]) + assert "out-count" not in out.columns + assert np.allclose(out["out-mean"].iloc[0], [2.0, 3.0]) + + +class TestDictToDf: + def setup_method(self): + (self.X,) = init_args([np.ones(3)], names=["X"]) + + def test_param_key_with_different_param_names(self): + def f(X, a=0): + return a + + def g(X, b=0): + return -b + + vset = build_vset("v", [f, g], param_dict=[{"a": [1]}, {"b": [2]}]) + df = dict_to_df(vset(self.X), param_key="v") + row_g = df[df["func-v"] == "g"].iloc[0] + assert row_g["b-v"] == "2" and pd.isna(row_g["a-v"]) + assert list(df.columns) == ["init-v", "func-v", "a-v", "b-v", "out"] + + def test_param_values_containing_equals(self): + def f(X, q="x"): + return q + + vset = build_vset("v", f, param_dict={"q": ["k=v"]}) + df = dict_to_df(vset(self.X), param_key="v") + assert df["q-v"].iloc[0] == "k=v" + + def test_plain_string_keys_give_a_clear_error(self): + vset = Vset("v", [np.sum], vfunc_keys=["s"]) + with pytest.raises(ValueError, match="name=value"): + dict_to_df(vset(self.X), param_key="v") + + +def test_cum_acc_by_uncertainty_rejects_class_predictions(): + preds = {("m",): np.array([0, 1, 1])} + with pytest.raises(ValueError, match="class probabilities"): + cum_acc_by_uncertainty(preds, preds, [0, 1, 1]) + + +class TestPCSPipeline: + def test_generate_names(self): + p = PCSPipeline( + [ + Vset("a", [np.abs, np.square], vfunc_keys=["abs", "sq"]), + Vset("b", [np.sum], vfunc_keys=["sum"]), + ] + ) + names = p.generate_names() + assert list(names.columns) == ["a", "b"] and len(names) == 2 + flat = p.generate_names(as_pandas=False) + assert len(flat) == 2 and flat[0][0].startswith("a_0_") + + def test_run_chains_steps_and_returns_outputs(self): + p = PCSPipeline( + [ + Vset("a", [np.abs, np.square], vfunc_keys=["abs", "sq"]), + Vset("b", [np.sum], vfunc_keys=["sum"]), + ] + ) + (x,) = init_args([np.array([-1.0, 2.0])], names=["x"]) + out = p.run(x) + assert sorted(values(out)) == [3.0, 5.0] + + +class TestVsetConstruction: + def test_dict_with_plain_keys(self): + from sklearn.linear_model import LogisticRegression + + vset = Vset("m", {"LR": LogisticRegression()}) + X, y = init_args([np.random.default_rng(0).normal(size=(20, 2)), [0, 1] * 10]) + vset.fit(X, y) + assert [k[0].value for k in vset.vfuncs] == ["LR"] + assert next(iter(vset.vfuncs))[0].origin == "m" + + def test_duplicate_keys_and_bad_containers(self): + with pytest.raises(ValueError, match="unique"): + Vset("m", [np.sum, np.mean], vfunc_keys=["s", "s"]) + with pytest.raises(TypeError, match="list or a dict"): + Vset("m", (np.sum, np.mean)) + + def test_build_vset_merges_identical_configurations(self): + def f(X, a=0): + return a + + vset = build_vset("v", f, param_dict=[{"a": [1, 2]}, {"a": [2, 3]}]) + assert len(vset) == 3 + + def test_predict_needs_a_predictor(self): + (x,) = init_args([np.ones(3)], names=["x"]) + vset = Vset("s", [np.sum], vfunc_keys=["s"]).fit(x) + with pytest.raises(AttributeError, match="'predict'"): + vset.predict(x) + + +def test_subkey_hash_matches_eq(): + from vflow import Subkey + + a, b = Subkey("x", "o", True), Subkey("x", "o", False) + assert a == b and hash(a) == hash(b) + assert (a,) in {(b,): 1} + + +class TestMlflowTracking: + def test_params_logged_and_runs_closed(self, tmp_path): + from mlflow.tracking import MlflowClient + + (x,) = init_args([np.array([-1.0, 2.0])], names=["x"]) + up = Vset("up", [np.abs, np.square], vfunc_keys=["abs", "sq"]) + met = Vset("met", [np.sum], vfunc_keys=["sum"], tracking_dir=str(tmp_path)) + met(up(x)) + client = MlflowClient(tracking_uri=str(tmp_path)) + runs = client.search_runs([met._exp_id]) + assert len(runs) == 2 + assert sorted(r.data.params["up"] for r in runs) == ["abs", "sq"] + assert {r.info.status for r in runs} == {"FINISHED"} + + def test_fit_with_tracking_does_not_crash(self, tmp_path): + from sklearn.linear_model import LogisticRegression + + X, y = init_args([np.random.default_rng(0).normal(size=(20, 2)), [0, 1] * 10]) + Vset( + "mm", [LogisticRegression()], vfunc_keys=["LR"], tracking_dir=str(tmp_path) + ).fit(X, y) + + +class TestCaching: + def test_redefined_function_is_not_served_stale(self, tmp_path): + (x,) = init_args([np.array([1.0, 2.0])], names=["x"]) + + def f(a): + return a + 1 + + first = values(Vset("s", [f], vfunc_keys=["f"], cache_dir=str(tmp_path))(x)) + + def f(a): # noqa: F811 - redefined on purpose, as in a notebook + return a + 100 + + second = values(Vset("s", [f], vfunc_keys=["f"], cache_dir=str(tmp_path))(x)) + assert np.array_equal(first[0], [2.0, 3.0]) + assert np.array_equal(second[0], [101.0, 102.0]) + + def test_lambdas_and_cache_hits(self, tmp_path): + (x,) = init_args([np.array([1.0, 2.0])], names=["x"]) + # count calls in a file: a closure over a mutable list would change the + # function's fingerprint after the first call + log = str(tmp_path / "calls.txt") + + def slow(a): + with open(log, "a") as f: + f.write("x") + return a * 2 + + vset = Vset("lam", [lambda a: a * 2, slow], cache_dir=str(tmp_path / "c")) + out1, out2 = values(vset(x)), values(vset(x)) + assert all(np.array_equal(u, v) for u, v in zip(out1, out2)) + with open(log) as f: + assert f.read() == "x", "the second call is served from the cache" + + +def test_lazy_async_feeds_lazy_sync(): + (x,) = init_args([np.array([1.0, 2.0])], names=["x"]) + up = Vset("up", [lambda a: a + 1], is_async=True, lazy=True) + down = Vset("down", [lambda a: a * 2], lazy=True) + (promise,) = values(down(up(x))) + assert np.array_equal(promise(), [4.0, 6.0]) + + +def test_async_detected_from_dict_of_async_vfuncs(): + from vflow import AsyncVfunc, Subkey + + (x,) = init_args([np.array([1.0, 2.0])], names=["x"]) + vset = Vset("s", {(Subkey("s", "s"),): AsyncVfunc("s", np.sum)}) + assert vset._async + assert values(vset(x)) == [3.0] diff --git a/tests/test_seeding.py b/tests/test_seeding.py new file mode 100644 index 0000000..1b55842 --- /dev/null +++ b/tests/test_seeding.py @@ -0,0 +1,86 @@ +from functools import partial + +import numpy as np +import pytest +from sklearn.datasets import make_classification +from sklearn.ensemble import RandomForestClassifier +from sklearn.utils import resample + +from vflow import build_vset, init_args + + +def draws(vset, X): + """Run every vfunc of `vset` on X and return the outputs in vfunc order.""" + (Xs,) = [vset(*init_args([X], names=["X"]), n_out=1)] + return [v for k, v in Xs.items() if k != "__prev__"] + + +class TestSeeding: + def setup_method(self): + self.X = np.arange(40).reshape(20, 2) + + def test_reps_differ_and_reproduce(self): + a = draws(build_vset("boot", resample, reps=4, seed=0), self.X) + assert len(a) == 4 + assert len({x.tobytes() for x in a}) == 4, "each rep draws its own sample" + b = draws(build_vset("boot", resample, reps=4, seed=0), self.X) + assert all(np.array_equal(x, y) for x, y in zip(a, b)), "same seed, same draws" + c = draws(build_vset("boot", resample, reps=4, seed=1), self.X) + assert not all(np.array_equal(x, y) for x, y in zip(a, c)) + + def test_seed_with_param_dict_and_class(self): + X, y = make_classification(n_samples=60, n_features=4, random_state=0) + vset = build_vset( + "RF", + RandomForestClassifier, + param_dict={"max_depth": [2, 3]}, + reps=2, + seed=7, + n_estimators=5, + ) + states = [vf.vfunc.random_state for vf in vset.vfuncs.values()] + assert len(states) == 4 and len(set(states)) == 4 + again = build_vset( + "RF", + RandomForestClassifier, + param_dict={"max_depth": [2, 3]}, + reps=2, + seed=7, + n_estimators=5, + ) + assert states == [vf.vfunc.random_state for vf in again.vfuncs.values()] + p1 = [vf.vfunc.fit(X, y).predict_proba(X) for vf in vset.vfuncs.values()] + p2 = [vf.vfunc.fit(X, y).predict_proba(X) for vf in again.vfuncs.values()] + assert all(np.array_equal(u, v) for u, v in zip(p1, p2)) + + def test_no_seed_is_unchanged(self): + vset = build_vset("boot", resample, reps=3) + assert all( + "random_state" not in vf.vfunc.keywords for vf in vset.vfuncs.values() + ) + + def test_seed_conflicts_and_unseedable_funcs(self): + with pytest.raises(ValueError, match="either `seed` or `random_state`"): + build_vset("boot", resample, reps=2, seed=0, random_state=3) + with pytest.raises(ValueError, match="either `seed` or `random_state`"): + build_vset("boot", resample, param_dict={"random_state": [1, 2]}, seed=0) + + def double(X): + return 2 * X + + with pytest.raises(ValueError, match="does not"): + build_vset("double", double, reps=2, seed=0) + + def test_partial_and_callable_object_names(self): + vset = build_vset( + "boot", partial(resample, replace=True), param_dict={"n_samples": [5]} + ) + assert next(iter(vset.vfuncs))[0].value[0] == "func=resample" + + class Scale: + def __call__(self, X, factor=1): + return factor * X + + vset = build_vset("scale", Scale(), param_dict={"factor": [2, 3]}) + names = [k[0].value[0] for k in vset.vfuncs] + assert names == ["func=Scale", "func=Scale"] diff --git a/vflow/helpers.py b/vflow/helpers.py index 9364c37..5d0e37b 100644 --- a/vflow/helpers.py +++ b/vflow/helpers.py @@ -1,6 +1,7 @@ """User-facing helper functions included at import vflow """ +import inspect from functools import partial from itertools import product from typing import Union @@ -38,11 +39,41 @@ def init_args(args_tuple: Union[tuple, list], names=None): return output_dicts +def _key_origin(vset: Vset) -> str: + """The origin of a Vset's vfunc keys, i.e. the column its outputs get in + dict_to_df: the Vset's name, or the original name for a filtered Vset.""" + for k in vset.vfuncs: + return k[0].origin + return vset.name + + +def _func_name(f) -> str: + """A readable name for `f`: a function's __name__, the wrapped function's + for a functools.partial, the class name for a callable object.""" + if isinstance(f, partial): + return _func_name(f.func) + return getattr(f, "__name__", None) or type(f).__name__ + + +def _accepts_random_state(f) -> bool: + """Whether `f` (a function, partial, class or callable object) can take a + `random_state` keyword argument. Returns True when it cannot be told.""" + try: + sig = inspect.signature(f) + except (TypeError, ValueError): + return True + params = sig.parameters.values() + return "random_state" in sig.parameters or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in params + ) + + def build_vset( name: str, func, param_dict=None, reps: int = 1, + seed: int = None, is_async: bool = False, output_matching: bool = False, lazy: bool = False, @@ -78,6 +109,15 @@ def build_vset( reps : int, optional The number of times to repeat `func` in the output Vset's vfuncs for each combination of the parameters in `param_dict`. + seed : int, optional + Makes randomized vfuncs reproducible. When given, every vfunc gets its + own `random_state`, derived from `seed` (with numpy's SeedSequence) and + passed to `func` as a keyword argument. So `reps` copies of e.g. + `sklearn.utils.resample` draw different samples, and building the Vset + again with the same `seed` and arguments draws the same ones. `func` + must accept a `random_state` argument, and `random_state` must not also + be set in `param_dict` or `**kwargs`. If None (the default), nothing is + seeded and repetitions use whatever randomness `func` uses by default. is_async : bool, optional If True, vfuncs are computed asynchronously. output_matching : bool, optional @@ -119,6 +159,28 @@ def build_vset( vfuncs = [] vkeys = [] + if seed is not None: + if "random_state" in kwargs or any( + pd is not None and "random_state" in pd for pd in pd_list + ): + raise ValueError( + "pass either `seed` or `random_state`, not both: `seed` sets a " + "different random_state for every vfunc" + ) + for f in f_list: + if not _accepts_random_state(f): + raise ValueError( + f"`seed` needs `func` to accept a `random_state` argument, " + f"and {_func_name(f)} does not" + ) + n_vfuncs = reps * sum( + int(np.prod([len(v) for v in (pd or {}).values()])) for pd in pd_list + ) + # one independent, reproducible stream per vfunc + seeds = iter( + int(s) for s in np.random.SeedSequence(seed).generate_state(n_vfuncs) + ) + for f, pd in zip(f_list, pd_list): if pd is None: pd = {} @@ -127,7 +189,7 @@ def build_vset( kwargs_tuples = product(*list(pd.values())) for tup in kwargs_tuples: kwargs_dict = {} - vkey_tup = (f"func={f.__name__}",) + vkey_tup = (f"func={_func_name(f)}",) for param_name, param_val in zip(list(pd.keys()), tup): kwargs_dict[param_name] = param_val vkey_tup += (f"{param_name}={param_val}",) @@ -140,17 +202,25 @@ def build_vset( vkeys.append((f"rep={i}",) + vkey_tup) else: vkeys.append(vkey_tup) + rep_kwargs = dict(kwargs_dict) + if seed is not None: + rep_kwargs["random_state"] = next(seeds) # check if func is a class if isinstance(f, type): # instantiate func - vfuncs.append(Vfunc(vfunc=f(**kwargs_dict), name=str(vkey_tup))) + vfuncs.append(Vfunc(vfunc=f(**rep_kwargs), name=str(vkey_tup))) else: # use partial to wrap func vfuncs.append( - Vfunc(vfunc=partial(f, **kwargs_dict), name=str(vkey_tup)) + Vfunc(vfunc=partial(f, **rep_kwargs), name=str(vkey_tup)) ) if all(pd is None for pd in pd_list) and reps == 1: vkeys = None + elif len(set(vkeys)) < len(vkeys): + # overlapping param_dicts repeat a configuration; keep one vfunc per + # key (the last, as a dict would), in first-seen order + merged = dict(zip(vkeys, vfuncs)) + vkeys, vfuncs = list(merged.keys()), list(merged.values()) return Vset( name, @@ -206,28 +276,29 @@ def filter_vset_by_metric( vsets = [vset, *vsets] vset_names = [] for vset_i in vsets: - if vset_i.name not in df.columns: + # outputs are labelled by the origin of the vfunc keys, which stays the + # original Vset's name after filtering (a filtered Vset is renamed) + col = _key_origin(vset_i) + if col not in df.columns: raise ValueError( - ( - f"{vset_i.name} should be one " - "of the columns of dict_to_df(metric_dict)" - ) + f"{col} should be one of the columns of dict_to_df(metric_dict)" ) - vset_names.append(vset_i.name) + vset_names.append(col) if len(filter_on) > 0: - filter_col = list(metric_dict.keys())[0][-1].origin + filter_col = [k for k in metric_dict if k != PREV_KEY][0][-1].origin df = df[df[filter_col].isin(filter_on)] if group: df = df.groupby(by=vset_names, as_index=False).mean(numeric_only=True) - if bigger_is_better: - df = df.sort_values(by="out", ascending=False) - else: - df = df.sort_values(by="out") - df = df.iloc[0:n_keep] + # stable sort: among tied values, the earlier row wins, every time + df = df.sort_values(by="out", ascending=not bigger_is_better, kind="stable") + # n_keep counts distinct vfunc combinations, not metric rows: one vfunc can + # top several rows when it was run on several upstream perturbations + df = df.drop_duplicates(subset=vset_names).iloc[0:n_keep] for i, vset_i in enumerate(vsets): vfuncs = vset_i.vfuncs - vfunc_filter = [str(name) for name in df[vset_i.name].to_numpy()] - new_vfuncs = {k: v for k, v in vfuncs.items() if str(v.name) in vfunc_filter} + keep = list(df[vset_names[i]]) + # match on the key the outputs are labelled with, not on Vfunc.name + new_vfuncs = {k: v for k, v in vfuncs.items() if k[0].value in keep} tracking_dir = None if vset_i._mlflow is None else mlflow.get_tracking_uri() new_vset = Vset( "filtered_" + vset_i.name, @@ -260,9 +331,10 @@ def cum_acc_by_uncertainty(mean_preds, std_preds, true_labels): Params ------ mean_preds: dict - mean predictions, output from Vset.predict_with_uncertainties + mean class probabilities per group, from + Vset.predict_proba(..., with_uncertainty=True) std_preds: dict - std predictions, output from Vset.predict_with_uncertainties + std of class probabilities per group, from the same call true_labels: dict or list-like TODO: generalize to multi-class classification @@ -274,7 +346,15 @@ def cum_acc_by_uncertainty(mean_preds, std_preds, true_labels): paired_preds = [ [d[k] for d in (mean_preds, std_preds)] for k in dict_keys(mean_preds) ] - mean_preds, std_preds = (np.array(p)[:, :, 1] for p in zip(*paired_preds)) + arrays = [np.array(p) for p in zip(*paired_preds)] + if arrays[0].ndim != 3: + raise ValueError( + "cum_acc_by_uncertainty needs class probabilities of shape " + "(n_obs, n_classes) per group, as from " + "Vset.predict_proba(..., with_uncertainty=True); got predictions " + f"of shape {arrays[0].shape[1:]}" + ) + mean_preds, std_preds = (a[:, :, 1] for a in arrays) if isinstance(true_labels, dict): true_labels = dict_data(true_labels) assert len(true_labels) == 1, "true_labels should have a single 1D vector entry" diff --git a/vflow/pipeline.py b/vflow/pipeline.py index 04d6632..7e79e47 100644 --- a/vflow/pipeline.py +++ b/vflow/pipeline.py @@ -28,16 +28,26 @@ def __init__(self, steps=None, cache_dir=None): self.memory = joblib.Memory(location=cache_dir) def run(self, *args, **kwargs): - """Runs the pipeline""" + """Runs the pipeline: each step is called on the previous step's outputs + (the first on `args`), and the last step's outputs are returned. + + A step that returns a tuple or list (e.g. one output dict per input) + passes them on as separate positional arguments. `kwargs` go to every + step. + """ run_step_cached = self.memory.cache(_run_step) + outputs = args for i, step in enumerate(self.steps): try: step_name = step.name except AttributeError: step_name = f"Step {i}" print(step_name) - _, fitted_step = run_step_cached(step, *args, **kwargs) + outputs, fitted_step = run_step_cached(step, *outputs, **kwargs) self.steps[i] = fitted_step + if not isinstance(outputs, (tuple, list)): + outputs = (outputs,) + return outputs[0] if len(outputs) == 1 else outputs def __getitem__(self, i): """Accesses ith step of pipeline""" @@ -47,17 +57,20 @@ def __len__(self): return len(self.steps) def generate_names(self, as_pandas=True): + """Names for every combination of vfuncs across the steps: a DataFrame + with one column per step, or a list of tuples.""" name_lists = [] - if as_pandas: - for step in self.steps: - name_lists.append([f"{i}_{str(mod)[:8]}" for i, mod in enumerate(step)]) - indexes = list(itertools.product(*name_lists)) - return pd.DataFrame(indexes, columns=[step.name for step in self.steps]) for step in self.steps: + # a Vset is not iterable; its vfuncs are + mods = list(step.vfuncs.values()) + prefix = "" if as_pandas else f"{step.name}_" name_lists.append( - [f"{step.name}_{i}_{str(mod)[:8]}" for i, mod in enumerate(step)] + [f"{prefix}{i}_{str(mod)[:8]}" for i, mod in enumerate(mods)] ) - return list(itertools.product(*name_lists)) + indexes = list(itertools.product(*name_lists)) + if as_pandas: + return pd.DataFrame(indexes, columns=[step.name for step in self.steps]) + return indexes def build_graph(node, draw=True): diff --git a/vflow/subkey.py b/vflow/subkey.py index 9467d35..a3ab6ba 100644 --- a/vflow/subkey.py +++ b/vflow/subkey.py @@ -80,5 +80,6 @@ def __repr__(self): return str(self.value) def __hash__(self): - """Mainly used for testing purposes.""" - return hash(self.value) ^ hash(self.origin) ^ hash(self.output_matching) + """Consistent with __eq__, which compares value and origin only: keys + that compare equal must hash equal or dict and set lookups miss them.""" + return hash(self.value) ^ hash(self.origin) diff --git a/vflow/utils.py b/vflow/utils.py index 61b460e..fdae9aa 100644 --- a/vflow/utils.py +++ b/vflow/utils.py @@ -229,20 +229,26 @@ def dict_to_df(d: dict, param_key=None): df = df.join(param_df) else: param_loc = df.columns.get_loc(param_key) - param_key_cols = [ - f"{p.split('=')[0]}-{param_key}" for p in param_keys[0] - ] - param_keys = [[s.split("=")[1] for s in t] for t in param_keys] - df = df.join(pd.DataFrame(param_keys)).drop(columns=param_key) - new_cols = df.columns[: len(cols) - 1].tolist() + param_key_cols - df = df.set_axis(new_cols, axis=1) - new_idx = list(range(len(new_cols))) - new_idx = ( - new_idx[:param_loc] - + new_idx[len(cols) - 1 :] - + new_idx[param_loc : len(cols) - 1] + # parse every row on its own: vfuncs built from different funcs + # can have different parameter names + rows = [] + for t in param_keys: + if isinstance(t, str) or not all( + isinstance(s, str) and "=" in s for s in t + ): + raise ValueError( + f"param_key={param_key!r} needs keys that are tuples " + "of 'name=value' strings, as build_vset(param_dict=...) " + f"makes; got {t!r}" + ) + # split on the first '=' only, so values may contain '=' + rows.append(dict(s.split("=", 1) for s in t)) + param_df = pd.DataFrame(rows, index=df.index) + param_df.columns = [f"{c}-{param_key}" for c in param_df.columns] + df = pd.concat( + [df.iloc[:, :param_loc], param_df, df.iloc[:, param_loc + 1 :]], + axis=1, ) - df = df.iloc[:, new_idx] return df @@ -286,6 +292,11 @@ def perturbation_stats( """ if func is None: func = ["count", "mean", "std"] + if isinstance(func, str): + func = [func] + # for scalar outputs, columns are prefixed only when a prefix is asked for, + # so existing callers keep their column names + prefix_scalar_cols = prefix is not None if prefix is None: prefix = wrt if isinstance(data, dict): @@ -301,7 +312,7 @@ def perturbation_stats( gb = df.groupby(group_by)[wrt] else: gb = df.groupby(lambda x: True)[wrt] - if (isinstance(func, list) and "mean" in func or "std" in func) and ( + if (isinstance(func, list) and ("mean" in func or "std" in func)) and ( type(df[wrt].iloc[0]) in [list, np.ndarray] ): # iterate the groups (same order as gb.groups); get_group with a single @@ -339,9 +350,13 @@ def perturbation_stats( wrt_stds.index.names = df_out.index.names df_out = df_out.join(wrt_stds) if "count" not in func: - df_out = df_out.drop(f"{prefix}-count") + df_out = df_out.drop(columns=f"{prefix}-count") else: df_out = gb.agg(func) + if isinstance(df_out, pd.Series): # a single callable gives a Series + df_out = df_out.to_frame(getattr(func, "__name__", wrt)) + if prefix_scalar_cols: + df_out.columns = [f"{prefix}-{c}" for c in df_out.columns] df_out = df_out.reindex(sorted(df_out.columns), axis=1) df_out.reset_index(inplace=True) if len(group_by) > 0: diff --git a/vflow/vfunc.py b/vflow/vfunc.py index 42950fc..fe9d0d1 100644 --- a/vflow/vfunc.py +++ b/vflow/vfunc.py @@ -89,6 +89,12 @@ def __call__(self): tmp_args[i] = tmp_args[i]() while isinstance(self.vfunc, VfuncPromise): self.vfunc = self.vfunc() + # an upstream lazy async Vset leaves Ray references; a remote call + # resolves them itself, but a plain function must get the values + if not isinstance(getattr(self.vfunc, "__self__", None), AsyncVfunc): + tmp_args = [ + ray.get(a) if isinstance(a, ray.ObjectRef) else a for a in tmp_args + ] self.value = self.vfunc(*tmp_args) self.called = True return self.value diff --git a/vflow/vset.py b/vflow/vset.py index 6f87965..f69ed02 100644 --- a/vflow/vset.py +++ b/vflow/vset.py @@ -2,7 +2,9 @@ Function arguments are each a list """ +import hashlib import os +import warnings from copy import deepcopy import joblib @@ -10,6 +12,11 @@ import ray from mlflow.tracking import MlflowClient +try: + import cloudpickle +except ImportError: # ray always ships a copy + from ray import cloudpickle + from vflow.subkey import Subkey from vflow.utils import ( PREV_KEY, @@ -67,7 +74,8 @@ def __init__( self._output_matching = output_matching self._lazy = lazy self._cache_dir = cache_dir - self._memory = joblib.Memory(self._cache_dir) + # verbose=0: by default joblib prints every call with its full arguments + self._memory = joblib.Memory(self._cache_dir, verbose=0) if tracking_dir is not None: if "://" not in str(tracking_dir): # a plain directory: MLflow 3 needs an explicit opt-in for its @@ -81,13 +89,27 @@ def __init__( self._exp_id = experiment.experiment_id else: self._mlflow = None + if not isinstance(vfuncs, (dict, list)): + raise TypeError( + f"vfuncs must be a list or a dict, not {type(vfuncs).__name__}" + ) # check if any of the vfuncs are AsyncVfuncs # if so, we'll make then all AsyncVfuncs later on - if not self._async and np.any([isinstance(vf, AsyncVfunc) for vf in vfuncs]): + funcs = vfuncs.values() if isinstance(vfuncs, dict) else vfuncs + if not self._async and any(isinstance(vf, AsyncVfunc) for vf in funcs): self._async = True if isinstance(vfuncs, dict): - self.vfuncs = vfuncs - elif isinstance(vfuncs, list): + # keys may already be tuples of Subkeys (e.g. from another Vset); + # plain keys like "LR" become this Vset's subkeys + self.vfuncs = { + ( + k + if isinstance(k, tuple) and all(isinstance(s, Subkey) for s in k) + else (self.__create_subkey(k),) + ): v + for k, v in vfuncs.items() + } + else: if vfunc_keys is not None: assert isinstance( vfunc_keys, list @@ -95,6 +117,11 @@ def __init__( assert len(vfuncs) == len( vfunc_keys ), "vfuncs list and vfunc_keys list do not have the same length" + if len(set(map(repr, vfunc_keys))) < len(vfunc_keys): + raise ValueError( + f"vfunc_keys must be unique; duplicates would silently " + f"drop vfuncs: {vfunc_keys}" + ) # TODO: how best to handle tuple subkeys? vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] else: @@ -139,8 +166,29 @@ def _apply_func(self, *args, out_dict: dict = None): if out_dict is None: out_dict = deepcopy(self.vfuncs) - apply_func_cached = self._memory.cache(_apply_func_cached) - out_dict = apply_func_cached(out_dict, self._async, self._lazy, *args) + cache_key = None + if self._cache_dir is not None: + # joblib hashes functions by name, so redefining a function would + # serve stale results, and it cannot hash lambdas at all. Key the + # cache on the functions' code and the inputs instead. + try: + cache_key = hashlib.sha256( + cloudpickle.dumps((out_dict, self._async, self._lazy, args)) + ).hexdigest() + except Exception as e: # noqa: BLE001 - fall back to no caching + warnings.warn( + f"Vset {self.name!r}: could not fingerprint the inputs for " + f"caching ({type(e).__name__}: {e}); running without the cache" + ) + if cache_key is None: + out_dict = _apply_func_cached(None, out_dict, self._async, self._lazy, args) + else: + apply_func_cached = self._memory.cache( + _apply_func_cached, ignore=["out_dict", "args"] + ) + out_dict = apply_func_cached( + cache_key, out_dict, self._async, self._lazy, args + ) prev = tuple() for arg in args: @@ -152,7 +200,9 @@ def _apply_func(self, *args, out_dict: dict = None): run_dict = {} # log subkeys as params and value as metric for k, v in out_dict.items(): - if k == PREV_KEY: + # only numeric scalars are metrics; fitted models, predictions + # and lazy promises are not logged + if k == PREV_KEY or not _is_numeric_scalar(v): continue origins = np.array([subk.origin for subk in k]) # ignore init origins and the last origin (this Vset) @@ -173,8 +223,11 @@ def _apply_func(self, *args, out_dict: dict = None): if np.sum(origins == param_name) > 1: occurence = np.sum(origins[:idx] == param_name) param_name = param_name + str(occurence) - self._mlflow.log_param(run_id, param_name, subkey.value) - self._mlflow.log_metric(run_id, k[-1].value, v) + self._mlflow.log_param(run_id, param_name, subkey.value) + self._mlflow.log_metric(run_id, k[-1].value, float(v)) + # close the runs, or they stay RUNNING forever + for run_id in run_dict.values(): + self._mlflow.set_terminated(run_id) return out_dict def fit(self, *args): @@ -214,6 +267,7 @@ def predict(self, *args, with_uncertainty: bool = False, group_by: list = None): for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict"): pred_dict[k] = v.predict + self._require_method(pred_dict, "predict") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -231,6 +285,7 @@ def predict_proba( for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict_proba"): pred_dict[k] = v.predict_proba + self._require_method(pred_dict, "predict_proba") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -283,6 +338,15 @@ def __len__(self): def __str__(self): return "Vset(" + self.name + ")" + def _require_method(self, method_dict, method): + """Raise rather than silently return no outputs when no fitted vfunc + has `method`.""" + if not method_dict: + raise AttributeError( + f"None of the fitted vfuncs in Vset {self.name!r} has a " + f"{method!r} method." + ) + def __create_subkey(self, value): """Helper function to construct `Subkey` with this Vset determining origin and output_matching @@ -290,19 +354,31 @@ def __create_subkey(self, value): return Subkey(value, self.name, self._output_matching) -def _apply_func_cached(out_dict: dict, is_async: bool, lazy: bool, *args): +def _is_numeric_scalar(v) -> bool: + """Whether `v` can be logged as an MLflow metric.""" + return isinstance(v, (int, float, np.integer, np.floating)) and not isinstance( + v, bool + ) + + +def _apply_func_cached( + cache_key, out_dict: dict, is_async: bool, lazy: bool, args: tuple +): """ Params ------ - *args: dict - Takes multiple dicts and combines them into one. - Then runs vfuncs on each item in combined dict. + cache_key: str or None + Fingerprint of the vfuncs and inputs; the only argument joblib hashes + when caching (out_dict and args are ignored by the cache). out_dict: dict The dictionary to pass to the matching function. is_async: bool If True, outputs are computed asynchronously. lazy: bool If True, outputs are evaluated lazily, i.e. outputs are `VfuncPromise`. + args: tuple of dict + Takes multiple dicts and combines them into one. + Then runs vfuncs on each item in combined dict. Returns ------- From b96cf6e21896e255b9b7d85db9f363261db273bc Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Thu, 15 Oct 2026 16:56:17 -0400 Subject: [PATCH 3/6] Docs: end index at References, drop Contributing from guides, rebuild Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Xy8Dg7fAqocHGzMqVCiSK --- docs/agents.html | 93 +- docs/helpers.html | 227 +++- docs/html.mako | 1 - docs/index.html | 2468 +------------------------------------ docs/pipeline.html | 105 +- docs/style_docs.py | 63 +- docs/subkey.html | 11 +- docs/utils.html | 91 +- docs/vflow-internals.html | 93 +- docs/vfunc.html | 13 +- docs/vset.html | 180 ++- 11 files changed, 694 insertions(+), 2651 deletions(-) diff --git a/docs/agents.html b/docs/agents.html index cc291a5..470fe17 100644 --- a/docs/agents.html +++ b/docs/agents.html @@ -323,7 +323,6 @@

7  Notes and limits

  • Notebooks

    @@ -352,7 +351,7 @@

    7  Notes and limits

    • -

  • API

    +

  • API

    • -

  • Functions

    +

  • Functions

    • -

  • Classes

    +

  • Classes

    diff --git a/docs/helpers.html b/docs/helpers.html index aba25a3..f69b584 100644 --- a/docs/helpers.html +++ b/docs/helpers.html @@ -55,6 +55,7 @@ 
    """User-facing helper functions included at import vflow
 """
 
+import inspect
 from functools import partial
 from itertools import product
 from typing import Union
@@ -92,11 +93,41 @@
     return output_dicts
 
 
+def _key_origin(vset: Vset) -> str:
+    """The origin of a Vset's vfunc keys, i.e. the column its outputs get in
+    dict_to_df: the Vset's name, or the original name for a filtered Vset."""
+    for k in vset.vfuncs:
+        return k[0].origin
+    return vset.name
+
+
+def _func_name(f) -> str:
+    """A readable name for `f`: a function's __name__, the wrapped function's
+    for a functools.partial, the class name for a callable object."""
+    if isinstance(f, partial):
+        return _func_name(f.func)
+    return getattr(f, "__name__", None) or type(f).__name__
+
+
+def _accepts_random_state(f) -> bool:
+    """Whether `f` (a function, partial, class or callable object) can take a
+    `random_state` keyword argument. Returns True when it cannot be told."""
+    try:
+        sig = inspect.signature(f)
+    except (TypeError, ValueError):
+        return True
+    params = sig.parameters.values()
+    return "random_state" in sig.parameters or any(
+        p.kind is inspect.Parameter.VAR_KEYWORD for p in params
+    )
+
+
 def build_vset(
     name: str,
     func,
     param_dict=None,
     reps: int = 1,
+    seed: int = None,
     is_async: bool = False,
     output_matching: bool = False,
     lazy: bool = False,
@@ -132,6 +163,15 @@
     reps : int, optional
         The number of times to repeat `func` in the output Vset's vfuncs for
         each combination of the parameters in `param_dict`.
+    seed : int, optional
+        Makes randomized vfuncs reproducible. When given, every vfunc gets its
+        own `random_state`, derived from `seed` (with numpy's SeedSequence) and
+        passed to `func` as a keyword argument. So `reps` copies of e.g.
+        `sklearn.utils.resample` draw different samples, and building the Vset
+        again with the same `seed` and arguments draws the same ones. `func`
+        must accept a `random_state` argument, and `random_state` must not also
+        be set in `param_dict` or `**kwargs`. If None (the default), nothing is
+        seeded and repetitions use whatever randomness `func` uses by default.
     is_async : bool, optional
         If True, vfuncs are computed asynchronously.
     output_matching : bool, optional
@@ -173,6 +213,28 @@
     vfuncs = []
     vkeys = []
 
+    if seed is not None:
+        if "random_state" in kwargs or any(
+            pd is not None and "random_state" in pd for pd in pd_list
+        ):
+            raise ValueError(
+                "pass either `seed` or `random_state`, not both: `seed` sets a "
+                "different random_state for every vfunc"
+            )
+        for f in f_list:
+            if not _accepts_random_state(f):
+                raise ValueError(
+                    f"`seed` needs `func` to accept a `random_state` argument, "
+                    f"and {_func_name(f)} does not"
+                )
+        n_vfuncs = reps * sum(
+            int(np.prod([len(v) for v in (pd or {}).values()])) for pd in pd_list
+        )
+        # one independent, reproducible stream per vfunc
+        seeds = iter(
+            int(s) for s in np.random.SeedSequence(seed).generate_state(n_vfuncs)
+        )
+
     for f, pd in zip(f_list, pd_list):
         if pd is None:
             pd = {}
@@ -181,7 +243,7 @@
         kwargs_tuples = product(*list(pd.values()))
         for tup in kwargs_tuples:
             kwargs_dict = {}
-            vkey_tup = (f"func={f.__name__}",)
+            vkey_tup = (f"func={_func_name(f)}",)
             for param_name, param_val in zip(list(pd.keys()), tup):
                 kwargs_dict[param_name] = param_val
                 vkey_tup += (f"{param_name}={param_val}",)
@@ -194,17 +256,25 @@
                     vkeys.append((f"rep={i}",) + vkey_tup)
                 else:
                     vkeys.append(vkey_tup)
+                rep_kwargs = dict(kwargs_dict)
+                if seed is not None:
+                    rep_kwargs["random_state"] = next(seeds)
                 # check if func is a class
                 if isinstance(f, type):
                     # instantiate func
-                    vfuncs.append(Vfunc(vfunc=f(**kwargs_dict), name=str(vkey_tup)))
+                    vfuncs.append(Vfunc(vfunc=f(**rep_kwargs), name=str(vkey_tup)))
                 else:
                     # use partial to wrap func
                     vfuncs.append(
-                        Vfunc(vfunc=partial(f, **kwargs_dict), name=str(vkey_tup))
+                        Vfunc(vfunc=partial(f, **rep_kwargs), name=str(vkey_tup))
                     )
     if all(pd is None for pd in pd_list) and reps == 1:
         vkeys = None
+    elif len(set(vkeys)) < len(vkeys):
+        # overlapping param_dicts repeat a configuration; keep one vfunc per
+        # key (the last, as a dict would), in first-seen order
+        merged = dict(zip(vkeys, vfuncs))
+        vkeys, vfuncs = list(merged.keys()), list(merged.values())
 
     return Vset(
         name,
@@ -260,28 +330,29 @@
     vsets = [vset, *vsets]
     vset_names = []
     for vset_i in vsets:
-        if vset_i.name not in df.columns:
+        # outputs are labelled by the origin of the vfunc keys, which stays the
+        # original Vset's name after filtering (a filtered Vset is renamed)
+        col = _key_origin(vset_i)
+        if col not in df.columns:
             raise ValueError(
-                (
-                    f"{vset_i.name} should be one "
-                    "of the columns of dict_to_df(metric_dict)"
-                )
+                f"{col} should be one of the columns of dict_to_df(metric_dict)"
             )
-        vset_names.append(vset_i.name)
+        vset_names.append(col)
     if len(filter_on) > 0:
-        filter_col = list(metric_dict.keys())[0][-1].origin
+        filter_col = [k for k in metric_dict if k != PREV_KEY][0][-1].origin
         df = df[df[filter_col].isin(filter_on)]
     if group:
         df = df.groupby(by=vset_names, as_index=False).mean(numeric_only=True)
-    if bigger_is_better:
-        df = df.sort_values(by="out", ascending=False)
-    else:
-        df = df.sort_values(by="out")
-    df = df.iloc[0:n_keep]
+    # stable sort: among tied values, the earlier row wins, every time
+    df = df.sort_values(by="out", ascending=not bigger_is_better, kind="stable")
+    # n_keep counts distinct vfunc combinations, not metric rows: one vfunc can
+    # top several rows when it was run on several upstream perturbations
+    df = df.drop_duplicates(subset=vset_names).iloc[0:n_keep]
     for i, vset_i in enumerate(vsets):
         vfuncs = vset_i.vfuncs
-        vfunc_filter = [str(name) for name in df[vset_i.name].to_numpy()]
-        new_vfuncs = {k: v for k, v in vfuncs.items() if str(v.name) in vfunc_filter}
+        keep = list(df[vset_names[i]])
+        # match on the key the outputs are labelled with, not on Vfunc.name
+        new_vfuncs = {k: v for k, v in vfuncs.items() if k[0].value in keep}
         tracking_dir = None if vset_i._mlflow is None else mlflow.get_tracking_uri()
         new_vset = Vset(
             "filtered_" + vset_i.name,
@@ -314,9 +385,10 @@
     Params
     ------
     mean_preds: dict
-        mean predictions, output from Vset.predict_with_uncertainties
+        mean class probabilities per group, from
+        Vset.predict_proba(..., with_uncertainty=True)
     std_preds: dict
-        std predictions, output from Vset.predict_with_uncertainties
+        std of class probabilities per group, from the same call
     true_labels: dict or list-like
 
     TODO: generalize to multi-class classification
@@ -328,7 +400,15 @@
     paired_preds = [
         [d[k] for d in (mean_preds, std_preds)] for k in dict_keys(mean_preds)
     ]
-    mean_preds, std_preds = (np.array(p)[:, :, 1] for p in zip(*paired_preds))
+    arrays = [np.array(p) for p in zip(*paired_preds)]
+    if arrays[0].ndim != 3:
+        raise ValueError(
+            "cum_acc_by_uncertainty needs class probabilities of shape "
+            "(n_obs, n_classes) per group, as from "
+            "Vset.predict_proba(..., with_uncertainty=True); got predictions "
+            f"of shape {arrays[0].shape[1:]}"
+        )
+    mean_preds, std_preds = (a[:, :, 1] for a in arrays)
     if isinstance(true_labels, dict):
         true_labels = dict_data(true_labels)
         assert len(true_labels) == 1, "true_labels should have a single 1D vector entry"
@@ -354,7 +434,7 @@
 
    Functions
    
 
    
 
    
-def build_vset(name: str, func, param_dict=None, reps: int = 1, is_async: bool = False, output_matching: bool = False, lazy: bool = False, cache_dir: str = None, tracking_dir: str = None, **kwargs) ‑> Vset
+def build_vset(name: str, func, param_dict=None, reps: int = 1, seed: int = None, is_async: bool = False, output_matching: bool = False, lazy: bool = False, cache_dir: str = None, tracking_dir: str = None, **kwargs) ‑> Vset
 
    
 
    
 
    Builds a new Vset by currying or instantiating callable func with all
@@ -384,6 +464,15 @@ 
    Parameters
    
 
    reps : int, optional
    
 
    The number of times to repeat func in the output Vset's vfuncs for
 each combination of the parameters in param_dict.
    
+
    seed : int, optional
    
+
    Makes randomized vfuncs reproducible. When given, every vfunc gets its
+own random_state, derived from seed (with numpy's SeedSequence) and
+passed to func as a keyword argument. So reps copies of e.g.
+sklearn.utils.resample draw different samples, and building the Vset
+again with the same seed and arguments draws the same ones. func
+must accept a random_state argument, and random_state must not also
+be set in param_dict or **kwargs. If None (the default), nothing is
+seeded and repetitions use whatever randomness func uses by default.
    
 
    is_async : bool, optional
    
 
    If True, vfuncs are computed asynchronously.
    
 
    output_matching : bool, optional
    
@@ -412,6 +501,7 @@ 
    Returns
    
     func,
     param_dict=None,
     reps: int = 1,
+    seed: int = None,
     is_async: bool = False,
     output_matching: bool = False,
     lazy: bool = False,
@@ -447,6 +537,15 @@ 
    Returns
    
     reps : int, optional
         The number of times to repeat `func` in the output Vset's vfuncs for
         each combination of the parameters in `param_dict`.
+    seed : int, optional
+        Makes randomized vfuncs reproducible. When given, every vfunc gets its
+        own `random_state`, derived from `seed` (with numpy's SeedSequence) and
+        passed to `func` as a keyword argument. So `reps` copies of e.g.
+        `sklearn.utils.resample` draw different samples, and building the Vset
+        again with the same `seed` and arguments draws the same ones. `func`
+        must accept a `random_state` argument, and `random_state` must not also
+        be set in `param_dict` or `**kwargs`. If None (the default), nothing is
+        seeded and repetitions use whatever randomness `func` uses by default.
     is_async : bool, optional
         If True, vfuncs are computed asynchronously.
     output_matching : bool, optional
@@ -488,6 +587,28 @@ 
    Returns
    
     vfuncs = []
     vkeys = []
 
+    if seed is not None:
+        if "random_state" in kwargs or any(
+            pd is not None and "random_state" in pd for pd in pd_list
+        ):
+            raise ValueError(
+                "pass either `seed` or `random_state`, not both: `seed` sets a "
+                "different random_state for every vfunc"
+            )
+        for f in f_list:
+            if not _accepts_random_state(f):
+                raise ValueError(
+                    f"`seed` needs `func` to accept a `random_state` argument, "
+                    f"and {_func_name(f)} does not"
+                )
+        n_vfuncs = reps * sum(
+            int(np.prod([len(v) for v in (pd or {}).values()])) for pd in pd_list
+        )
+        # one independent, reproducible stream per vfunc
+        seeds = iter(
+            int(s) for s in np.random.SeedSequence(seed).generate_state(n_vfuncs)
+        )
+
     for f, pd in zip(f_list, pd_list):
         if pd is None:
             pd = {}
@@ -496,7 +617,7 @@ 
    Returns
    
         kwargs_tuples = product(*list(pd.values()))
         for tup in kwargs_tuples:
             kwargs_dict = {}
-            vkey_tup = (f"func={f.__name__}",)
+            vkey_tup = (f"func={_func_name(f)}",)
             for param_name, param_val in zip(list(pd.keys()), tup):
                 kwargs_dict[param_name] = param_val
                 vkey_tup += (f"{param_name}={param_val}",)
@@ -509,17 +630,25 @@ 
    Returns
    
                     vkeys.append((f"rep={i}",) + vkey_tup)
                 else:
                     vkeys.append(vkey_tup)
+                rep_kwargs = dict(kwargs_dict)
+                if seed is not None:
+                    rep_kwargs["random_state"] = next(seeds)
                 # check if func is a class
                 if isinstance(f, type):
                     # instantiate func
-                    vfuncs.append(Vfunc(vfunc=f(**kwargs_dict), name=str(vkey_tup)))
+                    vfuncs.append(Vfunc(vfunc=f(**rep_kwargs), name=str(vkey_tup)))
                 else:
                     # use partial to wrap func
                     vfuncs.append(
-                        Vfunc(vfunc=partial(f, **kwargs_dict), name=str(vkey_tup))
+                        Vfunc(vfunc=partial(f, **rep_kwargs), name=str(vkey_tup))
                     )
     if all(pd is None for pd in pd_list) and reps == 1:
         vkeys = None
+    elif len(set(vkeys)) < len(vkeys):
+        # overlapping param_dicts repeat a configuration; keep one vfunc per
+        # key (the last, as a dict would), in first-seen order
+        merged = dict(zip(vkeys, vfuncs))
+        vkeys, vfuncs = list(merged.keys()), list(merged.values())
 
     return Vset(
         name,
@@ -541,9 +670,10 @@ 
    Returns
    
 sorted in increasing order of uncertainty
    
 
    Params
    
 
    mean_preds: dict
-mean predictions, output from Vset.predict_with_uncertainties
+mean class probabilities per group, from
+Vset.predict_proba(…, with_uncertainty=True)
 std_preds: dict
-std predictions, output from Vset.predict_with_uncertainties
+std of class probabilities per group, from the same call
 true_labels: dict or list-like
    
 
    TODO: generalize to multi-class classification
    
 
    
@@ -557,9 +687,10 @@ 
    Params
    
     Params
     ------
     mean_preds: dict
-        mean predictions, output from Vset.predict_with_uncertainties
+        mean class probabilities per group, from
+        Vset.predict_proba(..., with_uncertainty=True)
     std_preds: dict
-        std predictions, output from Vset.predict_with_uncertainties
+        std of class probabilities per group, from the same call
     true_labels: dict or list-like
 
     TODO: generalize to multi-class classification
@@ -571,7 +702,15 @@ 
    Params
    
     paired_preds = [
         [d[k] for d in (mean_preds, std_preds)] for k in dict_keys(mean_preds)
     ]
-    mean_preds, std_preds = (np.array(p)[:, :, 1] for p in zip(*paired_preds))
+    arrays = [np.array(p) for p in zip(*paired_preds)]
+    if arrays[0].ndim != 3:
+        raise ValueError(
+            "cum_acc_by_uncertainty needs class probabilities of shape "
+            "(n_obs, n_classes) per group, as from "
+            "Vset.predict_proba(..., with_uncertainty=True); got predictions "
+            f"of shape {arrays[0].shape[1:]}"
+        )
+    mean_preds, std_preds = (a[:, :, 1] for a in arrays)
     if isinstance(true_labels, dict):
         true_labels = dict_data(true_labels)
         assert len(true_labels) == 1, "true_labels should have a single 1D vector entry"
@@ -665,28 +804,29 @@ 
    Returns
    
     vsets = [vset, *vsets]
     vset_names = []
     for vset_i in vsets:
-        if vset_i.name not in df.columns:
+        # outputs are labelled by the origin of the vfunc keys, which stays the
+        # original Vset's name after filtering (a filtered Vset is renamed)
+        col = _key_origin(vset_i)
+        if col not in df.columns:
             raise ValueError(
-                (
-                    f"{vset_i.name} should be one "
-                    "of the columns of dict_to_df(metric_dict)"
-                )
+                f"{col} should be one of the columns of dict_to_df(metric_dict)"
             )
-        vset_names.append(vset_i.name)
+        vset_names.append(col)
     if len(filter_on) > 0:
-        filter_col = list(metric_dict.keys())[0][-1].origin
+        filter_col = [k for k in metric_dict if k != PREV_KEY][0][-1].origin
         df = df[df[filter_col].isin(filter_on)]
     if group:
         df = df.groupby(by=vset_names, as_index=False).mean(numeric_only=True)
-    if bigger_is_better:
-        df = df.sort_values(by="out", ascending=False)
-    else:
-        df = df.sort_values(by="out")
-    df = df.iloc[0:n_keep]
+    # stable sort: among tied values, the earlier row wins, every time
+    df = df.sort_values(by="out", ascending=not bigger_is_better, kind="stable")
+    # n_keep counts distinct vfunc combinations, not metric rows: one vfunc can
+    # top several rows when it was run on several upstream perturbations
+    df = df.drop_duplicates(subset=vset_names).iloc[0:n_keep]
     for i, vset_i in enumerate(vsets):
         vfuncs = vset_i.vfuncs
-        vfunc_filter = [str(name) for name in df[vset_i.name].to_numpy()]
-        new_vfuncs = {k: v for k, v in vfuncs.items() if str(v.name) in vfunc_filter}
+        keep = list(df[vset_names[i]])
+        # match on the key the outputs are labelled with, not on Vfunc.name
+        new_vfuncs = {k: v for k, v in vfuncs.items() if k[0].value in keep}
         tracking_dir = None if vset_i._mlflow is None else mlflow.get_tracking_uri()
         new_vset = Vset(
             "filtered_" + vset_i.name,
@@ -762,7 +902,6 @@ 
    Parameters

  • Notebooks

    diff --git a/docs/html.mako b/docs/html.mako index 96f3b0e..97cc2c7 100644 --- a/docs/html.mako +++ b/docs/html.mako @@ -296,7 +296,6 @@

  • Notebooks

    diff --git a/docs/index.html b/docs/index.html index 49a0147..afec4c5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -97,7 +97,7 @@

    Why use vflowWhy use vflowReferences

    year = {2022} } -
    - -Expand source code - -
    """
-.. include:: ../README.md
-"""
-
-from .helpers import (
-    build_vset,
-    cum_acc_by_uncertainty,
-    filter_vset_by_metric,
-    init_args,
-)
-from .pipeline import PCSPipeline, build_graph
-from .subkey import Subkey
-from .utils import (
-    apply_vfuncs,
-    base_dict,
-    combine_dicts,
-    combine_keys,
-    dict_data,
-    dict_keys,
-    dict_to_df,
-    init_step,
-    perturbation_stats,
-    sep_dicts,
-    to_list,
-    to_tuple,
-)
-from .vfunc import AsyncVfunc, Vfunc, VfuncPromise
-from .vset import Vset
-
-__all__ = [
-    # vflow.helpers
-    "init_args",
-    "build_vset",
-    "filter_vset_by_metric",
-    "cum_acc_by_uncertainty",
-    # vflow.pipeline
-    "PCSPipeline",
-    "build_graph",
-    # vflow.subkey
-    "Subkey",
-    # vflow.utils
-    "apply_vfuncs",
-    "base_dict",
-    "combine_dicts",
-    "combine_keys",
-    "dict_data",
-    "dict_keys",
-    "dict_to_df",
-    "init_step",
-    "perturbation_stats",
-    "sep_dicts",
-    "to_list",
-    "to_tuple",
-    # vflow.vfunc
-    "Vfunc",
-    "AsyncVfunc",
-    "VfuncPromise",
-    # vflow.vset
-    "Vset",
-]
    -
    - -
    -

    Sub-modules

    -
    -
    vflow.helpers
    -
    -

    User-facing helper functions included at import vflow

    -
    -
    vflow.pipeline
    -
    -

    Class that stores the entire pipeline of steps in a data-science workflow

    -
    -
    vflow.subkey
    -
    -

    Defines a parameter from some origin Vset

    -
    -
    vflow.utils
    -
    -

    Useful functions for converting between different types (dicts, lists, tuples, etc.)

    -
    -
    vflow.vfunc
    -
    -

    A perturbation that can be used as a step in a pipeline

    -
    -
    vflow.vset
    -
    -

    Set of vfuncs to be parallelized over in a pipeline. -Function arguments are each a list

    -
    -
    -
    -
    -
    -
    -

    Functions

    -
    -
    -def apply_vfuncs(vfuncs: dict, data_dict: dict, lazy: bool = False) -
    -
    -

    Apply a dictionary of functions vfuncs to each item of data_dict, -optionally returning a dictionary of VfuncPromise objects if lazy is True

    -

    Output keys are determined by applying combine_keys() to each pair of items from -vfuncs and data_dict. This function is used by all Vsets to apply functions.

    -

    Parameters

    -
    -
    vfuncs : dict
    -
    Dictionary of functions to apply to data_dict.
    -
    data_dict : dict
    -
    Dictionary of parameters to call each function in vfuncs.
    -
    lazy : bool (option), default False
    -
    If True, vfuncs are applied lazily, returning VfuncPromise -objects,
    -
    -

    Returns

    -
    -
    out_dict : dict
    -
    Output dictionary of applying vfuncs to data_dict.
    -
    -
    - -Expand source code - -
    def apply_vfuncs(vfuncs: dict, data_dict: dict, lazy: bool = False):
-    """Apply a dictionary of functions `vfuncs` to each item of `data_dict`,
-    optionally returning a dictionary of `vflow.vfunc.VfuncPromise` objects if `lazy` is True
-
-    Output keys are determined by applying `combine_keys` to each pair of items from
-    `vfuncs` and `data_dict`. This function is used by all Vsets to apply functions.
-
-    Parameters
-    ----------
-    vfuncs: dict
-        Dictionary of functions to apply to `data_dict`.
-    data_dict: dict
-        Dictionary of parameters to call each function in `vfuncs`.
-    lazy: bool (option), default False
-        If True, `vfuncs` are applied lazily, returning `vflow.vfunc.VfuncPromise`
-        objects,
-
-    Returns
-    -------
-    out_dict: dict
-        Output dictionary of applying `vfuncs` to `data_dict`.
-    """
-    out_dict = {}
-    for vf_k in vfuncs:
-        if len(data_dict) == 0:
-            func = deepcopy(vfuncs[vf_k])
-            if lazy:
-                out_dict[vf_k] = VfuncPromise(func)
-            else:
-                out_dict[vf_k] = func()
-        for data_k in data_dict:
-            if PREV_KEY in (vf_k, data_k):
-                continue
-
-            combined_key = combine_keys(data_k, vf_k)
-
-            if not len(combined_key) > 0:
-                continue
-
-            func = deepcopy(vfuncs[vf_k])
-            if lazy:
-                # return a promise
-                out_dict[combined_key] = VfuncPromise(func, *data_dict[data_k])
-            else:
-                data_list = list(data_dict[data_k])
-                for i, data in enumerate(data_list):
-                    if isinstance(data, VfuncPromise):
-                        data_list[i] = data()
-                    if isinstance(func, RayRemoteFun) and not isinstance(
-                        data_list[i], ray.ObjectRef
-                    ):
-                        # send data to Ray's remote object store
-                        data_list[i] = ray.put(data_list[i])
-                    elif isinstance(data_list[i], ray.ObjectRef):
-                        # this is not a remote function so get the data
-                        data_list[i] = ray.get(data_list[i])
-                out_dict[combined_key] = func(*data_list)
-
-    return out_dict
    -
    -
    -
    -def base_dict(d: dict) -
    -
    -

    Remove PREV_KEY from dict d if present

    -
    - -Expand source code - -
    def base_dict(d: dict):
-    """Remove PREV_KEY from dict d if present"""
-    return {k: v for k, v in d.items() if k != PREV_KEY}
    -
    -
    -
    -def build_graph(node, draw=True) -
    -
    -

    Helper function that just calls build_graph_recur with an empty graph

    -

    Parameters

    -
    -
    node : dict or Vset
    -
     
    -
    -

    Returns

    -
    -
    G : nx.Digraph()
    -
     
    -
    -
    - -Expand source code - -
    def build_graph(node, draw=True):
-    """Helper function that just calls build_graph_recur with an empty graph
-
-    Parameters
-    ----------
-    node: dict or Vset
-
-    Returns
-    -------
-    G: nx.Digraph()
-    """
-
-    def unnest_node(node):
-        """Unnest a node, if necessary (i.e., when node is a tuple)
-
-        Parameters
-        ----------
-        node: str, dict, Vset, or tuple
-
-        Returns
-        -------
-        unnested_node: str, Vset, or None
-        """
-        node_type = type(node)
-        if node_type is str or "Vset" in str(node_type):
-            return node
-        if node_type is tuple:
-            return unnest_node(node[0])
-        return None
-
-    def build_graph_recur(node, G):
-        """Builds a graph up using __prev__ and PREV_KEY pointers
-
-        Parameters
-        ----------
-        node: str, dict, Vset, or tuple
-        G: nx.Digraph()
-
-        Returns
-        -------
-        G: nx.Digraph()
-        """
-        # base case: reached starting node
-        if isinstance(node, str):
-            return G
-
-        # initial case: starting at dict
-        if isinstance(node, dict):
-            s_node = "End"
-            nodes_prev = node[PREV_KEY]
-            G.add_edge(nodes_prev[0], s_node)
-            for node_prev in nodes_prev[1:]:
-                G.add_edge(unnest_node(node_prev), nodes_prev[0])
-                G = build_graph_recur(node_prev, G)
-            return G
-
-        # main case: at a vfuncset
-        if "Vset" in str(type(node)):
-            if hasattr(node, PREV_KEY):
-                nodes_prev = getattr(node, PREV_KEY)
-                for node_prev in nodes_prev:
-                    G.add_edge(unnest_node(node_prev), node)
-                    G = build_graph_recur(node_prev, G)
-            return G
-
-        # nested prev key case
-        if isinstance(node, tuple):
-            func_node = unnest_node(node[0])
-            G = build_graph_recur(func_node, G)
-            for arg_node in node[1:]:
-                G.add_edge(unnest_node(arg_node), func_node)
-                G = build_graph_recur(arg_node, G)
-            return G
-
-        return G
-
-    G = nx.DiGraph()
-    G = build_graph_recur(node, G)
-    if draw:
-        nx.draw(G, with_labels=True, node_color="#CCCCCC")
-    return G
    -
    -
    -
    -def build_vset(name: str, func, param_dict=None, reps: int = 1, is_async: bool = False, output_matching: bool = False, lazy: bool = False, cache_dir: str = None, tracking_dir: str = None, **kwargs) ‑> Vset -
    -
    -

    Builds a new Vset by currying or instantiating callable func with all -combinations of parameters in param_dict and optional additional **kwargs. -If func and param_dict are lists, then the ith entry of func will be -curried with ith entry of param_dict. If only one of func or param_dict -is a list, the same func/param_dict will be curried for all entries in the -list. Vfuncs are named with param_dict items as tuples of -str("param_name=param_val").

    -

    Parameters

    -
    -
    name : str
    -
    A name for the output Vset.
    -
    func : callable or list[callable]
    -
    A callable to use as the base for Vfuncs in the output Vset. Can also be -a class object, in which case the class is immediately instantiated with -the parameter combinations from param_dict. Can also be a list of -callables, where the ith entry corresponds to param_dict or the ith -entry of param_dict (if param_dict is a list).
    -
    param_dict : dict[str, list] or list[dict[str, list]], optional
    -
    A dict with string keys corresponding to argument names of func and -entries which are lists of values to pass to func at run time (or when -instantiating func if it's a class object). Can also be a list of -dicts, where the ith dict entry corresponds to func or the ith entry -of func (if func is a list). If no parameters are required for the -ith function, the ith entry of param_dict can be None.
    -
    reps : int, optional
    -
    The number of times to repeat func in the output Vset's vfuncs for -each combination of the parameters in param_dict.
    -
    is_async : bool, optional
    -
    If True, vfuncs are computed asynchronously.
    -
    output_matching : bool, optional
    -
    If True, then output keys from Vset will be matched when used in other -Vsets.
    -
    cache_dir : str, optional
    -
    If provided, do caching and use cache_dir as the data store for -joblib.Memory.
    -
    tracking_dir : str, optional
    -
    If provided, use the mlflow.tracking API to log outputs as metrics with -parameters determined by input keys.
    -
    **kwargs
    -
    Additional fixed keyword arguments to pass to func.
    -
    -

    Returns

    -
    -
    new_vset : Vset
    -
     
    -
    -
    - -Expand source code - -
    def build_vset(
-    name: str,
-    func,
-    param_dict=None,
-    reps: int = 1,
-    is_async: bool = False,
-    output_matching: bool = False,
-    lazy: bool = False,
-    cache_dir: str = None,
-    tracking_dir: str = None,
-    **kwargs,
-) -> Vset:
-    """Builds a new Vset by currying or instantiating callable `func` with all
-    combinations of parameters in `param_dict` and optional additional `**kwargs`.
-    If `func` and `param_dict` are lists, then the ith entry of `func` will be
-    curried with ith entry of `param_dict`. If only one of `func` or `param_dict`
-    is a list, the same `func`/`param_dict` will be curried for all entries in the
-    list. Vfuncs are named with `param_dict` items as tuples of
-    str("param_name=param_val").
-
-    Parameters
-    ----------
-    name : str
-        A name for the output Vset.
-    func : callable or list[callable]
-        A callable to use as the base for Vfuncs in the output Vset. Can also be
-        a class object, in which case the class is immediately instantiated with
-        the parameter combinations from `param_dict`. Can also be a list of
-        callables, where the ith entry corresponds to `param_dict` or the ith
-        entry of `param_dict` (if `param_dict` is a list).
-    param_dict : dict[str, list] or list[dict[str, list]], optional
-        A dict with string keys corresponding to argument names of `func` and
-        entries which are lists of values to pass to `func` at run time (or when
-        instantiating `func` if it's a class object). Can also be a list of
-        dicts, where the ith dict entry corresponds to `func` or the ith entry
-        of `func` (if `func` is a list). If no parameters are required for the
-        ith function, the ith entry of `param_dict` can be `None`.
-    reps : int, optional
-        The number of times to repeat `func` in the output Vset's vfuncs for
-        each combination of the parameters in `param_dict`.
-    is_async : bool, optional
-        If True, vfuncs are computed asynchronously.
-    output_matching : bool, optional
-        If True, then output keys from Vset will be matched when used in other
-        Vsets.
-    cache_dir : str, optional
-        If provided, do caching and use `cache_dir` as the data store for
-        joblib.Memory.
-    tracking_dir : str, optional
-        If provided, use the mlflow.tracking API to log outputs as metrics with
-        parameters determined by input keys.
-    **kwargs
-        Additional fixed keyword arguments to pass to `func`.
-
-    Returns
-    -------
-    new_vset : vflow.vset.Vset
-
-    """
-    f_list = []
-    pd_list = []
-    if isinstance(func, list):
-        if isinstance(param_dict, list):
-            assert len(param_dict) == len(
-                func
-            ), "list of param_dicts must be same length as list of funcs"
-            f_list.extend(func)
-            pd_list.extend(param_dict)
-        else:
-            pd_list.extend([param_dict] * len(func))
-            f_list.extend(func)
-    elif isinstance(param_dict, list):
-        f_list.extend([func] * len(param_dict))
-        pd_list.extend(param_dict)
-    else:
-        f_list.append(func)
-        pd_list.append(param_dict)
-
-    vfuncs = []
-    vkeys = []
-
-    for f, pd in zip(f_list, pd_list):
-        if pd is None:
-            pd = {}
-        assert callable(f), "func must be callable"
-
-        kwargs_tuples = product(*list(pd.values()))
-        for tup in kwargs_tuples:
-            kwargs_dict = {}
-            vkey_tup = (f"func={f.__name__}",)
-            for param_name, param_val in zip(list(pd.keys()), tup):
-                kwargs_dict[param_name] = param_val
-                vkey_tup += (f"{param_name}={param_val}",)
-            # add additional fixed kwargs to kwargs_dict
-            for k, v in kwargs.items():
-                kwargs_dict[k] = v
-            for i in range(reps):
-                # add vfunc key to vkeys
-                if reps > 1:
-                    vkeys.append((f"rep={i}",) + vkey_tup)
-                else:
-                    vkeys.append(vkey_tup)
-                # check if func is a class
-                if isinstance(f, type):
-                    # instantiate func
-                    vfuncs.append(Vfunc(vfunc=f(**kwargs_dict), name=str(vkey_tup)))
-                else:
-                    # use partial to wrap func
-                    vfuncs.append(
-                        Vfunc(vfunc=partial(f, **kwargs_dict), name=str(vkey_tup))
-                    )
-    if all(pd is None for pd in pd_list) and reps == 1:
-        vkeys = None
-
-    return Vset(
-        name,
-        vfuncs,
-        is_async=is_async,
-        vfunc_keys=vkeys,
-        output_matching=output_matching,
-        lazy=lazy,
-        cache_dir=cache_dir,
-        tracking_dir=tracking_dir,
-    )
    -
    -
    -
    -def combine_dicts(*args: dict, base_case=True) -
    -
    -

    Combines any number of dictionaries into a single dictionary. Dictionaries -are combined left to right matching all keys according to combine_keys()

    -

    Parameters

    -
    -
    *args : dict
    -
    Dictionaries to recursively combine left to right.
    -
    -

    Returns

    -
    -
    combined_dict : dict
    -
    Combined dictionary.
    -
    -
    - -Expand source code - -
    def combine_dicts(*args: dict, base_case=True):
-    """Combines any number of dictionaries into a single dictionary. Dictionaries
-    are combined left to right matching all keys according to `combine_keys`
-
-    Parameters
-    ----------
-    *args: dict
-        Dictionaries to recursively combine left to right.
-
-    Returns
-    -------
-    combined_dict: dict
-        Combined dictionary.
-    """
-    n_args = len(args)
-    combined_dict = {}
-    if n_args == 0:
-        return combined_dict
-    if n_args == 1:
-        for k in args[0]:
-            # wrap the dict values in tuples; this is helpful so that when we
-            # pass the values to a vfunc fun in we can just use * expansion
-            if k != PREV_KEY:
-                combined_dict[k] = (args[0][k],)
-            else:
-                combined_dict[k] = args[0][k]
-        return combined_dict
-    if n_args == 2:
-        for k0 in args[0]:
-            for k1 in args[1]:
-                if PREV_KEY in (k0, k1):
-                    continue
-
-                combined_key = combine_keys(k0, k1)
-
-                if len(combined_key) > 0:
-                    if base_case:
-                        combined_dict[combined_key] = (args[0][k0], args[1][k1])
-                    else:
-                        combined_dict[combined_key] = args[0][k0] + (args[1][k1],)
-
-        return combined_dict
-    # combine the first two dicts and call recursively with remaining args
-    return combine_dicts(combine_dicts(args[0], args[1]), *args[2:], base_case=False)
    -
    -
    -
    -def combine_keys(left_key, right_key) -
    -
    -

    Combines left_key and right_key, attempting to match on any Subkey where -Subkey.is_matching() is True.

    -

    Returns an empty key on failed matches when -Subkey.mismatches() is True. Always filters on right_key -and returns combined_key with left_key prefix.

    -

    Parameters

    -
    -
    left_key : tuple
    -
    Left tuple key to combine.
    -
    right_key : tuple
    -
    Right tuple key to combine.
    -
    -

    Returns

    -
    -
    combined_key : tuple
    -
    Combined tuple key filtered according to Subkey.matches() rules, -which is empty according to Subkey.mismatches() rule.
    -
    -
    - -Expand source code - -
    def combine_keys(left_key, right_key):
-    """Combines `left_key` and `right_key`, attempting to match on any `Subkey` where
-    `vflow.subkey.Subkey.is_matching` is `True`.
-
-    Returns an empty key on failed matches when
-    `vflow.subkey.Subkey.mismatches` is `True`. Always filters on `right_key`
-    and returns `combined_key` with `left_key` prefix.
-
-    Parameters
-    ----------
-    left_key: tuple
-        Left tuple key to combine.
-    right_key: tuple
-        Right tuple key to combine.
-
-    Returns
-    -------
-    combined_key: tuple
-        Combined tuple key filtered according to `vflow.subkey.Subkey.matches` rules,
-        which is empty according to `vflow.subkey.Subkey.mismatches` rule.
-
-    """
-    if len(left_key) < len(right_key):
-        match_key = left_key
-        compare_key = right_key
-    else:
-        match_key = right_key
-        compare_key = left_key
-    match_subkeys = [subkey for subkey in match_key if subkey.is_matching()]
-    if len(match_subkeys) > 0:
-        matched_subkeys = []
-        for subkey in match_subkeys:
-            for c_subkey in compare_key:
-                if subkey.matches(c_subkey):
-                    matched_subkeys.append(subkey)
-                    break
-                if subkey.mismatches(c_subkey):
-                    # subkeys with same origin but different values are rejected
-                    return ()
-        if len(matched_subkeys) > 0:
-            # always filter on right key
-            filtered_key = tuple(
-                subkey for subkey in right_key if subkey not in matched_subkeys
-            )
-            combined_key = left_key + filtered_key
-            return combined_key
-        return left_key + right_key
-    return left_key + right_key
    -
    -
    -
    -def cum_acc_by_uncertainty(mean_preds, std_preds, true_labels) -
    -
    -

    Returns uncertainty and cumulative accuracy for grouped class predictions, -sorted in increasing order of uncertainty

    -

    Params

    -

    mean_preds: dict -mean predictions, output from Vset.predict_with_uncertainties -std_preds: dict -std predictions, output from Vset.predict_with_uncertainties -true_labels: dict or list-like

    -

    TODO: generalize to multi-class classification

    -
    - -Expand source code - -
    def cum_acc_by_uncertainty(mean_preds, std_preds, true_labels):
-    """Returns uncertainty and cumulative accuracy for grouped class predictions,
-    sorted in increasing order of uncertainty
-
-    Params
-    ------
-    mean_preds: dict
-        mean predictions, output from Vset.predict_with_uncertainties
-    std_preds: dict
-        std predictions, output from Vset.predict_with_uncertainties
-    true_labels: dict or list-like
-
-    TODO: generalize to multi-class classification
-    """
-    assert dict_keys(mean_preds) == dict_keys(
-        std_preds
-    ), "mean_preds and std_preds must share the same keys"
-    # match predictions on keys
-    paired_preds = [
-        [d[k] for d in (mean_preds, std_preds)] for k in dict_keys(mean_preds)
-    ]
-    mean_preds, std_preds = (np.array(p)[:, :, 1] for p in zip(*paired_preds))
-    if isinstance(true_labels, dict):
-        true_labels = dict_data(true_labels)
-        assert len(true_labels) == 1, "true_labels should have a single 1D vector entry"
-        true_labels = true_labels[0]
-    n_obs = len(mean_preds[0])
-    assert (
-        len(true_labels) == n_obs
-    ), f"true_labels has {len(true_labels)} obs. but should have same as predictions ({n_obs})"
-    sorted_idx = np.argsort(std_preds, axis=1)
-    correct_labels = np.take_along_axis(
-        np.around(mean_preds) - true_labels == 0, sorted_idx, 1
-    )
-    uncertainty = np.take_along_axis(std_preds, sorted_idx, 1)
-    cum_acc = np.cumsum(correct_labels, axis=1) / range(1, n_obs + 1)
-    return uncertainty, cum_acc, sorted_idx
    -
    -
    -
    -def dict_data(d: dict) -
    -
    -

    Returns a list containing all data in dict d

    -
    - -Expand source code - -
    def dict_data(d: dict):
-    """Returns a list containing all data in dict d"""
-    return list(base_dict(d).values())
    -
    -
    -
    -def dict_keys(d: dict) -
    -
    -

    Returns a list containing all keys in dict d

    -
    - -Expand source code - -
    def dict_keys(d: dict):
-    """Returns a list containing all keys in dict d"""
-    return list(base_dict(d).keys())
    -
    -
    -
    -def dict_to_df(d: dict, param_key=None) -
    -
    -

    Converts a dictionary with tuple keys -into a pandas DataFrame, optionally seperating -parameters in param_key if not None

    -

    Parameters

    -
    -
    d : dict
    -
    Output dictionary with tuple keys from a Vset.
    -
    param_key : str (optional), default None
    -
    Name of parameter to seperate into multiple columns.
    -
    -

    Returns

    -
    -
    df : pandas.DataFrame
    -
    A DataFrame with d tuple keys seperated into columns.
    -
    -
    - -Expand source code - -
    def dict_to_df(d: dict, param_key=None):
-    """Converts a dictionary with tuple keys
-    into a pandas DataFrame, optionally seperating
-    parameters in `param_key` if not None
-
-    Parameters
-    ----------
-    d: dict
-        Output dictionary with tuple keys from a Vset.
-    param_key: str (optional), default None
-        Name of parameter to seperate into multiple columns.
-
-    Returns
-    -------
-    df: pandas.DataFrame
-        A DataFrame with `d` tuple keys seperated into columns.
-    """
-    d_copy = {tuple(sk.value for sk in k): d[k] for k in d if k != PREV_KEY}
-    df = pd.Series(d_copy).reset_index()
-    if len(d_copy.keys()) > 0:
-        key_list = list(d.keys())
-        subkey_list = key_list[0] if key_list[0] != PREV_KEY else key_list[1]
-        cols = [sk.origin for sk in subkey_list] + ["out"]
-        # set each init col to init-{next_vfunc_set}
-        cols = [
-            c if c != "init" else init_step(idx, cols) for idx, c in enumerate(cols)
-        ]
-        df = df.set_axis(cols, axis=1)
-        if param_key:
-            param_keys = df[
-                param_key
-            ].tolist()  # pylint: disable=unsubscriptable-object
-            if param_key == "out" and hasattr(param_keys[0], "__iter__"):
-                param_df = pd.DataFrame(param_keys)
-                param_df.columns = [f"{param_key}-{col}" for col in param_df.columns]
-                df = df.join(param_df)
-            else:
-                param_loc = df.columns.get_loc(param_key)
-                param_key_cols = [
-                    f"{p.split('=')[0]}-{param_key}" for p in param_keys[0]
-                ]
-                param_keys = [[s.split("=")[1] for s in t] for t in param_keys]
-                df = df.join(pd.DataFrame(param_keys)).drop(columns=param_key)
-                new_cols = df.columns[: len(cols) - 1].tolist() + param_key_cols
-                df = df.set_axis(new_cols, axis=1)
-                new_idx = list(range(len(new_cols)))
-                new_idx = (
-                    new_idx[:param_loc]
-                    + new_idx[len(cols) - 1 :]
-                    + new_idx[param_loc : len(cols) - 1]
-                )
-                df = df.iloc[:, new_idx]
-    return df
    -
    -
    -
    -def filter_vset_by_metric(metric_dict: dict, vset: Vset, *vsets: Vset, n_keep: int = 1, bigger_is_better: bool = True, filter_on=None, group: bool = False) ‑> Vset | list -
    -
    -

    Returns a new Vset by filtering vset.vfuncs based on values in filter_dict.

    -

    Parameters

    -
    -
    metric_dict : dict
    -
    output from a Vset, typically with metrics or other numeric values to use when -filtering vset.vfuncs
    -
    vset : Vset
    -
    a Vsets
    -
    *vsets : Vset
    -
    zero or more additional Vsets
    -
    n_keep : int (optional)
    -
    number of entries to keep from vset.vfuncs
    -
    bigger_is_better : bool (optional)
    -
    if True, then the top n_keep largest values are retained
    -
    filter_on : list[str] (optional)
    -
    if there are multiple metrics in metric_dict, you can specify a subset -to consider
    -
    group : bool (optional)
    -
    if True, average metrics after grouping values in metric_dict by the -input Vset names
    -
    -

    Returns

    -
    -
    *new_vset : Vset
    -
    Copies of the input Vsets but with Vfuncs filtered based on metrics
    -
    -
    - -Expand source code - -
    def filter_vset_by_metric(
-    metric_dict: dict,
-    vset: Vset,
-    *vsets: Vset,
-    n_keep: int = 1,
-    bigger_is_better: bool = True,
-    filter_on=None,
-    group: bool = False,
-) -> Union[Vset, list]:
-    """Returns a new Vset by filtering `vset.vfuncs` based on values in filter_dict.
-
-    Parameters
-    ----------
-    metric_dict: dict
-        output from a Vset, typically with metrics or other numeric values to use when
-        filtering `vset.vfuncs`
-    vset: Vset
-        a Vsets
-    *vsets: Vset
-        zero or more additional Vsets
-    n_keep: int (optional)
-        number of entries to keep from `vset.vfuncs`
-    bigger_is_better: bool (optional)
-        if True, then the top `n_keep` largest values are retained
-    filter_on: list[str] (optional)
-        if there are multiple metrics in `metric_dict`, you can specify a subset
-        to consider
-    group: bool (optional)
-        if True, average metrics after grouping values in `metric_dict` by the
-        input Vset names
-
-    Returns
-    -------
-    *new_vset : Vset
-        Copies of the input Vsets but with Vfuncs filtered based on metrics
-    """
-    if filter_on is None:
-        filter_on = []
-    df = dict_to_df(metric_dict)
-    vsets = [vset, *vsets]
-    vset_names = []
-    for vset_i in vsets:
-        if vset_i.name not in df.columns:
-            raise ValueError(
-                (
-                    f"{vset_i.name} should be one "
-                    "of the columns of dict_to_df(metric_dict)"
-                )
-            )
-        vset_names.append(vset_i.name)
-    if len(filter_on) > 0:
-        filter_col = list(metric_dict.keys())[0][-1].origin
-        df = df[df[filter_col].isin(filter_on)]
-    if group:
-        df = df.groupby(by=vset_names, as_index=False).mean(numeric_only=True)
-    if bigger_is_better:
-        df = df.sort_values(by="out", ascending=False)
-    else:
-        df = df.sort_values(by="out")
-    df = df.iloc[0:n_keep]
-    for i, vset_i in enumerate(vsets):
-        vfuncs = vset_i.vfuncs
-        vfunc_filter = [str(name) for name in df[vset_i.name].to_numpy()]
-        new_vfuncs = {k: v for k, v in vfuncs.items() if str(v.name) in vfunc_filter}
-        tracking_dir = None if vset_i._mlflow is None else mlflow.get_tracking_uri()
-        new_vset = Vset(
-            "filtered_" + vset_i.name,
-            new_vfuncs,
-            is_async=vset_i._async,
-            output_matching=vset_i._output_matching,
-            lazy=vset_i._lazy,
-            cache_dir=vset_i._cache_dir,
-            tracking_dir=tracking_dir,
-        )
-        setattr(
-            new_vset,
-            FILTER_PREV_KEY,
-            (
-                metric_dict[PREV_KEY],
-                vset_i,
-            ),
-        )
-        setattr(new_vset, PREV_KEY, getattr(new_vset, FILTER_PREV_KEY))
-        vsets[i] = new_vset
-    if len(vsets) == 1:
-        return vsets[0]
-    return vsets
    -
    -
    -
    -def init_args(args_tuple: tuple | list, names=None) -
    -
    -

    Converts tuple of arguments to a list of dicts

    -

    Parameters

    -
    -
    names : list-like (optional), default None
    -
    given names for each of the arguments in the tuple
    -
    -
    - -Expand source code - -
    def init_args(args_tuple: Union[tuple, list], names=None):
-    """Converts tuple of arguments to a list of dicts
-
-    Parameters
-    ----------
-    names: list-like (optional), default None
-        given names for each of the arguments in the tuple
-    """
-    if names is None:
-        names = ["start"] * len(args_tuple)
-    else:
-        assert len(names) == len(
-            args_tuple
-        ), "names should be same length as args_tuple"
-    output_dicts = []
-    for i, _ in enumerate(args_tuple):
-        output_dicts.append(
-            {
-                (Subkey(names[i], "init"),): args_tuple[i],
-                PREV_KEY: ("init",),
-            }
-        )
-    return output_dicts
    -
    -
    -
    -def init_step(idx, cols) -
    -
    -

    Helper function to find init suffix -in a column

    -

    Parameters

    -
    -
    idx : int
    -
    Index of 'init' column in cols.
    -
    cols : list[str]
    -
    List of column names.
    -
    -
    - -Expand source code - -
    def init_step(idx, cols):
-    """Helper function to find init suffix
-    in a column
-
-    Parameters
-    ----------
-    idx: int
-        Index of 'init' column in cols.
-    cols: list[str]
-        List of column names.
-    """
-    for i in range(idx, len(cols)):
-        if cols[i] != "init":
-            return "init-" + cols[i]
-    return None
    -
    -
    -
    -def perturbation_stats(data: pandas.DataFrame | dict, *group_by: str, wrt: str = 'out', func=None, prefix: str = None, split: bool = False) -
    -
    -

    Compute statistics for wrt in data, conditional on group_by

    -

    Parameters

    -
    -
    data : Union[pandas.DataFrame, dict]
    -
    DataFrame, as from calling dict_to_df() on an output dict from a Vset, -or the output dict itself.
    -
    *group_by : str
    -
    Vset names in data to group on. If none provided, treats everything as one big -group.
    -
    wrt : str (optional)
    -
    Column name in data or dict_to_df()(data) on which to compute statistics. -Defaults to 'out', the values of the original Vset output dict.
    -
    func : function, str, list or dict (optional), default None
    -
    A list of functions or function names to use for computing -statistics, analogous to the parameter of the same name in -pandas.core.groupby.DataFrameGroupBy.aggregate. If None, defaults to -['count', 'mean', 'std'].
    -
    prefix : str (optional), default None
    -
    A string to prefix to new columns in output DataFrame. If None, -uses the value of wrt.
    -
    split : bool (optional), default False
    -
    If True and wrt in data has list or numpy.ndarray entries, will -attempt to split the entries into multiple columns for the output.
    -
    -

    Returns

    -
    -
    df : pandas.DataFrame
    -
    A DataFrame with summary statistics on wrt.
    -
    -
    - -Expand source code - -
    def perturbation_stats(
-    data: Union[pd.DataFrame, dict],
-    *group_by: str,
-    wrt: str = "out",
-    func=None,
-    prefix: str = None,
-    split: bool = False,
-):
-    """Compute statistics for `wrt` in `data`, conditional on `group_by`
-
-    Parameters
-    ----------
-    data: Union[pandas.DataFrame, dict]
-        DataFrame, as from calling `dict_to_df` on an output dict from a Vset,
-        or the output dict itself.
-    *group_by: str
-        Vset names in `data` to group on. If none provided, treats everything as one big
-        group.
-    wrt: str (optional)
-        Column name in `data` or `dict_to_df(data)` on which to compute statistics.
-        Defaults to `'out'`, the values of the original Vset output dict.
-    func: function, str, list or dict (optional), default None
-        A list of functions or function names to use for computing
-        statistics, analogous to the parameter of the same name in
-        pandas.core.groupby.DataFrameGroupBy.aggregate. If `None`, defaults to
-        `['count', 'mean', 'std']`.
-    prefix: str (optional), default None
-        A string to prefix to new columns in output DataFrame. If `None`,
-        uses the value of `wrt`.
-    split: bool (optional), default False
-        If `True` and `wrt` in `data` has `list` or `numpy.ndarray` entries, will
-        attempt to split the entries into multiple columns for the output.
-
-    Returns
-    -------
-    df: pandas.DataFrame
-        A DataFrame with summary statistics on `wrt`.
-    """
-    if func is None:
-        func = ["count", "mean", "std"]
-    if prefix is None:
-        prefix = wrt
-    if isinstance(data, dict):
-        df = dict_to_df(data)
-    else:
-        df = data
-    group_by = list(group_by)
-    if len(group_by) == 1:
-        # a bare column name keeps scalar group keys (a one-item list gives
-        # tuple keys in pandas 4)
-        gb = df.groupby(group_by[0])[wrt]
-    elif len(group_by) > 1:
-        gb = df.groupby(group_by)[wrt]
-    else:
-        gb = df.groupby(lambda x: True)[wrt]
-    if (isinstance(func, list) and "mean" in func or "std" in func) and (
-        type(df[wrt].iloc[0]) in [list, np.ndarray]
-    ):
-        # iterate the groups (same order as gb.groups); get_group with a single
-        # group key changed meaning in pandas 3
-        wrt_arrays = [np.stack(d.tolist()) for _, d in gb]
-        n_cols = wrt_arrays[0].shape[1]
-        df_out = pd.DataFrame(gb.agg("count"))
-        df_out.columns = [f"{prefix}-count"]
-        if "mean" in func:
-            if split:
-                col_means = [arr.mean(axis=0) for arr in wrt_arrays]
-                wrt_means = pd.DataFrame(
-                    col_means,
-                    columns=[f"{prefix}{i}-mean" for i in range(n_cols)],
-                    index=gb.groups.keys(),
-                )
-            else:
-                col_means = [{f"{prefix}-mean": arr.mean(axis=0)} for arr in wrt_arrays]
-                wrt_means = pd.DataFrame(col_means, index=gb.groups.keys())
-            wrt_means.index.names = df_out.index.names
-            df_out = df_out.join(wrt_means)
-        if "std" in func:
-            if split:
-                col_stds = [arr.std(axis=0, ddof=1) for arr in wrt_arrays]
-                wrt_stds = pd.DataFrame(
-                    col_stds,
-                    columns=[f"{prefix}{i}-std" for i in range(n_cols)],
-                    index=gb.groups.keys(),
-                )
-            else:
-                col_stds = [
-                    {f"{prefix}-std": arr.std(axis=0, ddof=1)} for arr in wrt_arrays
-                ]
-                wrt_stds = pd.DataFrame(col_stds, index=gb.groups.keys())
-            wrt_stds.index.names = df_out.index.names
-            df_out = df_out.join(wrt_stds)
-        if "count" not in func:
-            df_out = df_out.drop(f"{prefix}-count")
-    else:
-        df_out = gb.agg(func)
-    df_out = df_out.reindex(sorted(df_out.columns), axis=1)
-    df_out.reset_index(inplace=True)
-    if len(group_by) > 0:
-        return df_out.sort_values(group_by[0])
-    return df_out
    -
    -
    -
    -def sep_dicts(d: dict, n_out: int = 1, keys=None) -
    -
    -

    Converts dictionary with value being saved as an iterable into multiple dictionaries

    -

    Assumes every value has same length n_out

    -

    Parameters

    -
    -
    d : dict
    -
    Dictionary with iterable values to be converted.
    -
    n_out : int, default 1
    -
    The number of dictionaries to separate d into.
    -
    keys : list-like, default None
    -
    Optional list of keys to use in output dicts.
    -
    -

    Returns

    -
    -
    sep_dicts_list : list
    -
    List of seperated dictionaries.
    -
    -

    Examples

    -
    >>> sep_dicts({k1: (x1, y1), k2: (x2, y2), ...,  '__prev__': p})
-[{k1: x1, k2: x2, ..., '__prev__': p}, {k1: y1, k2: y2, ..., '__prev__': p}]
-
    -
    - -Expand source code - -
    def sep_dicts(d: dict, n_out: int = 1, keys=None):
-    """Converts dictionary with value being saved as an iterable into multiple dictionaries
-
-    Assumes every value has same length n_out
-
-    Parameters
-    ----------
-    d: dict
-        Dictionary with iterable values to be converted.
-    n_out: int, default 1
-        The number of dictionaries to separate d into.
-    keys: list-like, default None
-        Optional list of keys to use in output dicts.
-
-    Returns
-    -------
-    sep_dicts_list: list
-        List of seperated dictionaries.
-
-    Examples
-    --------
-    >>> sep_dicts({k1: (x1, y1), k2: (x2, y2), ...,  '__prev__': p})
-    [{k1: x1, k2: x2, ..., '__prev__': p}, {k1: y1, k2: y2, ..., '__prev__': p}]
-    """
-    if keys is None:
-        keys = []
-    if len(keys) > 0 and len(keys) != n_out:
-        raise ValueError(f"keys should be empty or have length n_out={n_out}")
-    # empty dict -- return empty dict
-    if n_out <= 1:
-        return d
-    # try separating dict into multiple dicts
-    sep_dicts_id = str(uuid4())  # w/ high prob, uuid4 is unique
-    sep_dicts_list = [{} for _ in range(n_out)]
-    for key, value in d.items():
-        if key != PREV_KEY:
-            for i in range(n_out):
-                # assumes the correct sub-key for item i is in the i-th position
-                if len(keys) == 0:
-                    new_key = (key[i],) + key[n_out:]
-                else:
-                    new_sub = Subkey(
-                        value=keys[i], origin=key[-1].origin + "-" + str(i)
-                    )
-                    new_key = (new_sub,) + key
-                new_key[-1].sep_dicts_id = sep_dicts_id
-                if isinstance(value, VfuncPromise):
-                    # return a promise to get the value at index i of the
-                    # original promise
-                    value_i = VfuncPromise(lambda v, x: v[x], value, i)
-                else:
-                    value_i = value[i]
-                sep_dicts_list[i][new_key] = value_i
-
-    return sep_dicts_list
    -
    -
    -
    -def to_list(tup: tuple) -
    -
    -

    Convert from tuple to packed list

    -

    Allows us to call function with arguments in a loop

    -

    Parameters

    -
    -
    tup : tuple
    -
    tuple of objects to convert to packed list
    -
    -

    Raises

    -
    -
    ValueError
    -
    If passed uneven number of arguments without a list. Please wrap your args in a list.
    -
    -

    Examples

    -
    >>> to_list(([x1, x2, x3], [y1, y2, y3]))
-[[x1, y1], [x2, y2], [x3, y3]]
->>> to_list(([x1], [y1]))
-[[x1, y1]]
->>> to_list(([x1, x2, x3], ))
-[[x1], [x2], [x3]]
->>> to_list((x1, ))
-[[x1]]
->>> to_list((x1, y1))
-[[x1, y1]]
->>> to_list((x1, x2, x3, y1, y2, y3))
-[[x1, y1], [x2, y2], [x3, y3]]
-
    -
    - -Expand source code - -
    def to_list(tup: tuple):
-    """Convert from tuple to packed list
-
-    Allows us to call function with arguments in a loop
-
-    Parameters
-    ----------
-    tup: tuple
-        tuple of objects to convert to packed list
-
-    Raises
-    ------
-    ValueError
-        If passed uneven number of arguments without a list. Please wrap your args in a list.
-
-    Examples
-    --------
-    >>> to_list(([x1, x2, x3], [y1, y2, y3]))
-    [[x1, y1], [x2, y2], [x3, y3]]
-    >>> to_list(([x1], [y1]))
-    [[x1, y1]]
-    >>> to_list(([x1, x2, x3], ))
-    [[x1], [x2], [x3]]
-    >>> to_list((x1, ))
-    [[x1]]
-    >>> to_list((x1, y1))
-    [[x1, y1]]
-    >>> to_list((x1, x2, x3, y1, y2, y3))
-    [[x1, y1], [x2, y2], [x3, y3]]
-    """
-    n_tup = len(tup)
-    if n_tup == 0:
-        return []
-    if not isinstance(tup[0], list):
-        # the first element is data
-        if n_tup == 1:
-            return [list(tup)]
-        if n_tup % 2 != 0:
-            raise ValueError(
-                "Don't know how to handle uneven number of args "
-                "without a list. Please wrap your args in a list."
-            )
-        # assume first half of args is input and second half is outcome
-        return [list(el) for el in zip(tup[: (n_tup // 2)], tup[(n_tup // 2) :])]
-    if n_tup == 1:
-        return [[x] for x in tup[0]]
-    n_mods = len(tup[0])
-    lists_packed = [[] for _ in range(n_mods)]
-    for i in range(n_mods):
-        for j in range(n_tup):
-            lists_packed[i].append(tup[j][i])
-    return lists_packed
    -
    -
    -
    -def to_tuple(lists: list) -
    -
    -

    Convert from lists to unpacked tuple

    -

    Allows us to write X, y = to_tuple([[x1, y1], [x2, y2], [x3, y3]])

    -

    Parameters

    -
    -
    lists : list
    -
    list of objects to convert to unpacked tuple
    -
    -

    Examples

    -
    >>> to_tuple([[x1, y1], [x2, y2], [x3, y3]])
-([x1, x2, x3], [y1, y2, y3])
->>> to_tuple([[x1, y1]])
-([x1], [y1])
->>> to_tuple([m1, m2, m3])
-[m1, m2, m3]
-
    -
    - -Expand source code - -
    def to_tuple(lists: list):
-    """Convert from lists to unpacked tuple
-
-    Allows us to write `X, y = to_tuple([[x1, y1], [x2, y2], [x3, y3]])`
-
-    Parameters
-    ----------
-    lists: list
-        list of objects to convert to unpacked tuple
-
-    Examples
-    --------
-    >>> to_tuple([[x1, y1], [x2, y2], [x3, y3]])
-    ([x1, x2, x3], [y1, y2, y3])
-    >>> to_tuple([[x1, y1]])
-    ([x1], [y1])
-    >>> to_tuple([m1, m2, m3])
-    [m1, m2, m3]
-    """
-    n_mods = len(lists)
-    if n_mods <= 1:
-        return lists
-    if not isinstance(lists[0], list):
-        return lists
-    n_tup = len(lists[0])
-    tup = [[] for _ in range(n_tup)]
-    for i in range(n_mods):
-        for j in range(n_tup):
-            tup[j].append(lists[i][j])
-    return tuple(tup)
    -
    -
    -
    -
    -
    -

    Classes

    -
    -
    -class AsyncVfunc -(name: str = '', vfunc=<function AsyncVfunc.<lambda>>) -
    -
    -

    An asynchronous version of the Vfunc class.

    -
    - -Expand source code - -
    class AsyncVfunc:
-    """An asynchronous version of the Vfunc class."""
-
-    def __init__(self, name: str = "", vfunc=lambda x: x):
-        self.name = name
-        if isinstance(vfunc, Vfunc):
-            self.vfunc = vfunc.vfunc
-        else:
-            assert hasattr(vfunc, "fit") or callable(
-                vfunc
-            ), "vfunc must be an object with a fit method or a callable"
-            self.vfunc = vfunc
-
-    def fit(self, *args, **kwargs):
-        """This function fits params for this vfunc"""
-        if hasattr(self.vfunc, "fit"):
-            return _remote_fun.remote(self.vfunc.fit, *args, **kwargs)
-        return _remote_fun.remote(self.vfunc, *args, **kwargs)
-
-    def transform(self, *args, **kwargs):
-        """This function transforms its input in some way"""
-        if hasattr(self.vfunc, "transform"):
-            return _remote_fun.remote(self.vfunc.transform, *args, **kwargs)
-        return _remote_fun.remote(self.vfunc, *args, **kwargs)
-
-    def __call__(self, *args, **kwargs):
-        """This should decide what to call"""
-        return self.fit(*args, **kwargs)
    -
    -

    Methods

    -
    -
    -def fit(self, *args, **kwargs) -
    -
    -

    This function fits params for this vfunc

    -
    - -Expand source code - -
    def fit(self, *args, **kwargs):
-    """This function fits params for this vfunc"""
-    if hasattr(self.vfunc, "fit"):
-        return _remote_fun.remote(self.vfunc.fit, *args, **kwargs)
-    return _remote_fun.remote(self.vfunc, *args, **kwargs)
    -
    -
    -
    -def transform(self, *args, **kwargs) -
    -
    -

    This function transforms its input in some way

    -
    - -Expand source code - -
    def transform(self, *args, **kwargs):
-    """This function transforms its input in some way"""
-    if hasattr(self.vfunc, "transform"):
-        return _remote_fun.remote(self.vfunc.transform, *args, **kwargs)
-    return _remote_fun.remote(self.vfunc, *args, **kwargs)
    -
    -
    -
    -
    -
    -class PCSPipeline -(steps=None, cache_dir=None) -
    -
    -

    Parameters

    -
    -
    steps : list
    -
    a list of Vset instances
    -
    cache_dir : str, default=None
    -
    The directory to use as data store by joblib. If None, won't do -caching.
    -
    -
    - -Expand source code - -
    class PCSPipeline:
-    def __init__(self, steps=None, cache_dir=None):
-        """
-        Parameters
-        ----------
-        steps: list
-            a list of Vset instances
-        cache_dir: str, default=None
-            The directory to use as data store by `joblib`. If None, won't do
-            caching.
-        """
-        if steps is None:
-            steps = []
-        self.steps = steps
-        # set up the cache
-        self.memory = joblib.Memory(location=cache_dir)
-
-    def run(self, *args, **kwargs):
-        """Runs the pipeline"""
-        run_step_cached = self.memory.cache(_run_step)
-        for i, step in enumerate(self.steps):
-            try:
-                step_name = step.name
-            except AttributeError:
-                step_name = f"Step {i}"
-            print(step_name)
-            _, fitted_step = run_step_cached(step, *args, **kwargs)
-            self.steps[i] = fitted_step
-
-    def __getitem__(self, i):
-        """Accesses ith step of pipeline"""
-        return self.steps[i]
-
-    def __len__(self):
-        return len(self.steps)
-
-    def generate_names(self, as_pandas=True):
-        name_lists = []
-        if as_pandas:
-            for step in self.steps:
-                name_lists.append([f"{i}_{str(mod)[:8]}" for i, mod in enumerate(step)])
-            indexes = list(itertools.product(*name_lists))
-            return pd.DataFrame(indexes, columns=[step.name for step in self.steps])
-        for step in self.steps:
-            name_lists.append(
-                [f"{step.name}_{i}_{str(mod)[:8]}" for i, mod in enumerate(step)]
-            )
-        return list(itertools.product(*name_lists))
    -
    -

    Methods

    -
    -
    -def generate_names(self, as_pandas=True) -
    -
    -
    -
    - -Expand source code - -
    def generate_names(self, as_pandas=True):
-    name_lists = []
-    if as_pandas:
-        for step in self.steps:
-            name_lists.append([f"{i}_{str(mod)[:8]}" for i, mod in enumerate(step)])
-        indexes = list(itertools.product(*name_lists))
-        return pd.DataFrame(indexes, columns=[step.name for step in self.steps])
-    for step in self.steps:
-        name_lists.append(
-            [f"{step.name}_{i}_{str(mod)[:8]}" for i, mod in enumerate(step)]
-        )
-    return list(itertools.product(*name_lists))
    -
    -
    -
    -def run(self, *args, **kwargs) -
    -
    -

    Runs the pipeline

    -
    - -Expand source code - -
    def run(self, *args, **kwargs):
-    """Runs the pipeline"""
-    run_step_cached = self.memory.cache(_run_step)
-    for i, step in enumerate(self.steps):
-        try:
-            step_name = step.name
-        except AttributeError:
-            step_name = f"Step {i}"
-        print(step_name)
-        _, fitted_step = run_step_cached(step, *args, **kwargs)
-        self.steps[i] = fitted_step
    -
    -
    -
    -
    -
    -class Subkey -(value, origin: str, output_matching: bool = False) -
    -
    -

    Parameters

    -
    -
    value : Any
    -
    subkey value corresponding to a Vset vfunc
    -
    origin : str
    -
    name of the origin Vset of this Subkey
    -
    output_matching : bool (optional), default False
    -
    inherited from the Vset where the Subkey is created
    -
    -
    - -Expand source code - -
    class Subkey:
-    def __init__(self, value, origin: str, output_matching: bool = False):
-        """
-        Parameters
-        ----------
-        value: Any
-            subkey value corresponding to a Vset vfunc
-        origin: str
-            name of the origin Vset of this Subkey
-        output_matching: bool (optional), default False
-            inherited from the Vset where the Subkey is created
-        """
-        self.value = value
-        self.origin = origin
-        self.output_matching = output_matching
-        # sep_dicts_id identifies the particular call to sep_dicts() that this
-        # key's dictionary went through (if any).
-        self.sep_dicts_id = None
-
-    def is_matching(self):
-        """Checks if subkey should be matched in other Vsets"""
-        return self.output_matching or self.sep_dicts_id is not None
-
-    def matches_sep_dict_id(self, other: object):
-        """Helper to match Subkey by _sep_dict_id"""
-        if isinstance(other, self.__class__):
-            return (
-                self.sep_dicts_id is not None
-                and self.sep_dicts_id == other.sep_dicts_id
-            )
-        return False
-
-    def matches(self, other: object):
-        """When Subkey matching is required, determines if this Subkey is compatible
-        with another, meaning that the origins and values match, and either the
-        _sep_dicts_id matches or both Subkeys have _output_matching True.
-        """
-        if isinstance(other, self.__class__):
-            # they're both matching
-            cond0 = self.is_matching() and other.is_matching()
-            # value and origins match
-            cond1 = self.value == other.value and self.origin == other.origin
-            # sep_dicts_id matches
-            cond2 = self.sep_dicts_id == other.sep_dicts_id or (
-                self.output_matching and other.output_matching
-            )
-            return cond0 and cond1 and cond2
-        return False
-
-    def mismatches(self, other: object):
-        """When Subkey matching is required, determines if this Subkey and another are
-        a bad match, meaning either:
-
-        1. output_matching is True, origin is same, value is different
-        2. output_matching is False, sep_dicts_id is same and not None, origin
-           is same, value is different
-        """
-        if isinstance(other, self.__class__):
-            # one of the two keys is output_matching
-            cond0 = self.output_matching or other.output_matching
-            # neither key is output_matching but sep_dict_ids not None and match
-            cond1 = not cond0 and self.matches_sep_dict_id(other)
-            # origins match and values mismatch
-            cond2 = self.origin == other.origin and self.value != other.value
-            return (cond0 or cond1) and cond2
-        return True
-
-    def __eq__(self, other: object):
-        """Mainly used for testing purposes."""
-        if isinstance(other, self.__class__):
-            # value and origins match
-            return self.value == other.value and self.origin == other.origin
-        return False
-
-    def __repr__(self):
-        return str(self.value)
-
-    def __hash__(self):
-        """Mainly used for testing purposes."""
-        return hash(self.value) ^ hash(self.origin) ^ hash(self.output_matching)
    -
    -

    Methods

    -
    -
    -def is_matching(self) -
    -
    -

    Checks if subkey should be matched in other Vsets

    -
    - -Expand source code - -
    def is_matching(self):
-    """Checks if subkey should be matched in other Vsets"""
-    return self.output_matching or self.sep_dicts_id is not None
    -
    -
    -
    -def matches(self, other: object) -
    -
    -

    When Subkey matching is required, determines if this Subkey is compatible -with another, meaning that the origins and values match, and either the -_sep_dicts_id matches or both Subkeys have _output_matching True.

    -
    - -Expand source code - -
    def matches(self, other: object):
-    """When Subkey matching is required, determines if this Subkey is compatible
-    with another, meaning that the origins and values match, and either the
-    _sep_dicts_id matches or both Subkeys have _output_matching True.
-    """
-    if isinstance(other, self.__class__):
-        # they're both matching
-        cond0 = self.is_matching() and other.is_matching()
-        # value and origins match
-        cond1 = self.value == other.value and self.origin == other.origin
-        # sep_dicts_id matches
-        cond2 = self.sep_dicts_id == other.sep_dicts_id or (
-            self.output_matching and other.output_matching
-        )
-        return cond0 and cond1 and cond2
-    return False
    -
    -
    -
    -def matches_sep_dict_id(self, other: object) -
    -
    -

    Helper to match Subkey by _sep_dict_id

    -
    - -Expand source code - -
    def matches_sep_dict_id(self, other: object):
-    """Helper to match Subkey by _sep_dict_id"""
-    if isinstance(other, self.__class__):
-        return (
-            self.sep_dicts_id is not None
-            and self.sep_dicts_id == other.sep_dicts_id
-        )
-    return False
    -
    -
    -
    -def mismatches(self, other: object) -
    -
    -

    When Subkey matching is required, determines if this Subkey and another are -a bad match, meaning either:

    -
      -
    1. output_matching is True, origin is same, value is different
      2. -
    2. output_matching is False, sep_dicts_id is same and not None, origin -is same, value is different
      3. -
    -
    - -Expand source code - -
    def mismatches(self, other: object):
-    """When Subkey matching is required, determines if this Subkey and another are
-    a bad match, meaning either:
-
-    1. output_matching is True, origin is same, value is different
-    2. output_matching is False, sep_dicts_id is same and not None, origin
-       is same, value is different
-    """
-    if isinstance(other, self.__class__):
-        # one of the two keys is output_matching
-        cond0 = self.output_matching or other.output_matching
-        # neither key is output_matching but sep_dict_ids not None and match
-        cond1 = not cond0 and self.matches_sep_dict_id(other)
-        # origins match and values mismatch
-        cond2 = self.origin == other.origin and self.value != other.value
-        return (cond0 or cond1) and cond2
-    return True
    -
    -
    -
    -
    -
    -class Vfunc -(name: str = '', vfunc=<function Vfunc.<lambda>>) -
    -
    -

    Vfunc is basically a function along with a name attribute. -It may support a "fit" function, but may also just have a "transform" function. -If none of these is supported, it need only be a function

    -
    - -Expand source code - -
    class Vfunc:
-    """Vfunc is basically a function along with a name attribute.
-    It may support a "fit" function, but may also just have a "transform" function.
-    If none of these is supported, it need only be a function
-    """
-
-    def __init__(self, name: str = "", vfunc=lambda x: x):
-        assert hasattr(vfunc, "fit") or callable(
-            vfunc
-        ), "vfunc must be an object with a fit method or a callable"
-        self.name = name
-        self.vfunc = vfunc
-
-    def fit(self, *args, **kwargs):
-        """This function fits params for this vfunc"""
-        if hasattr(self.vfunc, "fit"):
-            return self.vfunc.fit(*args, **kwargs)
-        return self.vfunc(*args, **kwargs)
-
-    def transform(self, *args, **kwargs):
-        """This function transforms its input in some way"""
-        if hasattr(self.vfunc, "transform"):
-            return self.vfunc.transform(*args, **kwargs)
-        return self.vfunc(*args, **kwargs)
-
-    def __call__(self, *args, **kwargs):
-        """This should decide what to call"""
-        return self.fit(*args, **kwargs)
    -
    -

    Methods

    -
    -
    -def fit(self, *args, **kwargs) -
    -
    -

    This function fits params for this vfunc

    -
    - -Expand source code - -
    def fit(self, *args, **kwargs):
-    """This function fits params for this vfunc"""
-    if hasattr(self.vfunc, "fit"):
-        return self.vfunc.fit(*args, **kwargs)
-    return self.vfunc(*args, **kwargs)
    -
    -
    -
    -def transform(self, *args, **kwargs) -
    -
    -

    This function transforms its input in some way

    -
    - -Expand source code - -
    def transform(self, *args, **kwargs):
-    """This function transforms its input in some way"""
-    if hasattr(self.vfunc, "transform"):
-        return self.vfunc.transform(*args, **kwargs)
-    return self.vfunc(*args, **kwargs)
    -
    -
    -
    -
    -
    -class VfuncPromise -(vfunc: , *args) -
    -
    -

    A Vfunc promise used for lazy evaluation.

    -
    - -Expand source code - -
    class VfuncPromise:
-    """A Vfunc promise used for lazy evaluation."""
-
-    def __init__(self, vfunc: callable, *args):
-        self.vfunc = vfunc
-        self.args = args
-        self.called = False
-        self.value = None
-
-    def __call__(self):
-        """This should decide what to call"""
-        if self.called:
-            return self.value
-        tmp_args = []
-        for i, arg in enumerate(self.args):
-            tmp_args.append(arg)
-            while isinstance(tmp_args[i], VfuncPromise):
-                tmp_args[i] = tmp_args[i]()
-        while isinstance(self.vfunc, VfuncPromise):
-            self.vfunc = self.vfunc()
-        self.value = self.vfunc(*tmp_args)
-        self.called = True
-        return self.value
-
-    def _get_value(self):
-        if isinstance(self(), ray.ObjectRef):
-            self.value = ray.get(self.value)
-        return self.value
-
-    def transform(self, *args):
-        """This function transforms its input in some way"""
-        return self._get_value().transform(*args)
-
-    def predict(self, *args):
-        """This function calls predict on its inputs"""
-        return self._get_value().predict(*args)
-
-    def predict_proba(self, *args):
-        """This function calls predict_proba on its inputs"""
-        return self._get_value().predict_proba(*args)
-
-    def __repr__(self):
-        if self.called:
-            return f"Fulfilled VfuncPromise({self.value})"
-        return f"Unfulfilled VfuncPromise(func={self.vfunc}, args={self.args})"
    -
    -

    Methods

    -
    -
    -def predict(self, *args) -
    -
    -

    This function calls predict on its inputs

    -
    - -Expand source code - -
    def predict(self, *args):
-    """This function calls predict on its inputs"""
-    return self._get_value().predict(*args)
    -
    -
    -
    -def predict_proba(self, *args) -
    -
    -

    This function calls predict_proba on its inputs

    -
    - -Expand source code - -
    def predict_proba(self, *args):
-    """This function calls predict_proba on its inputs"""
-    return self._get_value().predict_proba(*args)
    -
    -
    -
    -def transform(self, *args) -
    -
    -

    This function transforms its input in some way

    -
    - -Expand source code - -
    def transform(self, *args):
-    """This function transforms its input in some way"""
-    return self._get_value().transform(*args)
    -
    -
    -
    -
    -
    -class Vset -(name: str, vfuncs, vfunc_keys: list = None, is_async: bool = False, output_matching: bool = False, lazy: bool = False, cache_dir: str = None, tracking_dir: str = None) -
    -
    -

    Parameters

    -
    -
    name : str
    -
    Name of this Vset.
    -
    vfuncs : list or dict
    -
    Dictionary of functions that we want to associate with
    -
    vfunc_keys : list (optional)
    -
    List of names corresponding to each vfunc
    -
    is_async : bool (optional)
    -
    If True, vfuncs are computed asynchronously
    -
    output_matching : bool (optional)
    -
    If True, then output keys from this Vset will be matched when used -in other Vsets
    -
    lazy : bool (optional)
    -
    If True, then vfuncs are evaluated lazily, i.e. outputs are vset.vfunc.VfuncPromise
    -
    cache_dir : str (optional)
    -
    If provided, do caching and use cache_dir as the data store for -joblib.Memory.
    -
    tracking_dir : str (optional)
    -
    If provided, use the mlflow.tracking api to log outputs as metrics -with params determined by input keys.
    -
    -
    - -Expand source code - -
    class Vset:
-    def __init__(
-        self,
-        name: str,
-        vfuncs,
-        vfunc_keys: list = None,
-        is_async: bool = False,
-        output_matching: bool = False,
-        lazy: bool = False,
-        cache_dir: str = None,
-        tracking_dir: str = None,
-    ):
-        """
-        Parameters
-        ----------
-        name: str
-            Name of this Vset.
-        vfuncs: list or dict
-            Dictionary of functions that we want to associate with
-        vfunc_keys: list (optional)
-            List of names corresponding to each vfunc
-        is_async: bool (optional)
-            If True, `vfuncs` are computed asynchronously
-        output_matching: bool (optional)
-            If True, then output keys from this Vset will be matched when used
-            in other Vsets
-        lazy: bool (optional)
-            If True, then vfuncs are evaluated lazily, i.e. outputs are `vset.vfunc.VfuncPromise`
-        cache_dir: str (optional)
-            If provided, do caching and use `cache_dir` as the data store for
-            `joblib.Memory`.
-        tracking_dir: str (optional)
-            If provided, use the `mlflow.tracking` api to log outputs as metrics
-            with params determined by input keys.
-
-        """
-        self.name = name
-        self._fitted = False
-        self.fitted_vfuncs = None  # outputs
-        self._async = is_async
-        self._output_matching = output_matching
-        self._lazy = lazy
-        self._cache_dir = cache_dir
-        self._memory = joblib.Memory(self._cache_dir)
-        if tracking_dir is not None:
-            if "://" not in str(tracking_dir):
-                # a plain directory: MLflow 3 needs an explicit opt-in for its
-                # file-based tracking store (respects a value the user has set)
-                os.environ.setdefault("MLFLOW_ALLOW_FILE_STORE", "true")
-            self._mlflow = MlflowClient(tracking_uri=tracking_dir)
-            experiment = self._mlflow.get_experiment_by_name(name=self.name)
-            if experiment is None:
-                self._exp_id = self._mlflow.create_experiment(name=self.name)
-            else:
-                self._exp_id = experiment.experiment_id
-        else:
-            self._mlflow = None
-        # check if any of the vfuncs are AsyncVfuncs
-        # if so, we'll make then all AsyncVfuncs later on
-        if not self._async and np.any([isinstance(vf, AsyncVfunc) for vf in vfuncs]):
-            self._async = True
-        if isinstance(vfuncs, dict):
-            self.vfuncs = vfuncs
-        elif isinstance(vfuncs, list):
-            if vfunc_keys is not None:
-                assert isinstance(
-                    vfunc_keys, list
-                ), "vfuncs passed as list but vfunc_keys is not a list"
-                assert len(vfuncs) == len(
-                    vfunc_keys
-                ), "vfuncs list and vfunc_keys list do not have the same length"
-                # TODO: how best to handle tuple subkeys?
-                vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys]
-            else:
-                vfunc_keys = [
-                    (self.__create_subkey(f"{name}_{i}"),) for i in range(len(vfuncs))
-                ]
-            # convert vfunc keys to singleton tuples
-            self.vfuncs = dict(zip(vfunc_keys, vfuncs))
-        # if needed, wrap the vfuncs in the Vfunc or AsyncVfunc class
-        for k, v in self.vfuncs.items():
-            if self._async:
-                if not isinstance(v, AsyncVfunc):
-                    self.vfuncs[k] = AsyncVfunc(k[0], v)
-            elif not isinstance(v, Vfunc):
-                self.vfuncs[k] = Vfunc(k[0], v)
-
-    def _apply_func(self, *args, out_dict: dict = None):
-        """Apply functions in out_dict to combined args dict
-
-        Optionally logs output Subkeys and values as params and metrics using
-        `mlflow.tracking` if this Vset has a `_tracking_dir`.
-
-        Parameters
-        ----------
-        *args: dict
-            Takes multiple dicts and combines them into one.
-            Then runs vfuncs on each item in combined dict.
-        out_dict: dict (optional), default None
-            The dictionary to pass to the matching function. If None, defaults to self.vfuncs.
-
-        Returns
-        -------
-        out_dict: dict
-            Dictionary with items being determined by functions in vfunc set.
-            Functions and input dictionaries are currently matched using a cartesian matching format.
-
-        Examples
-        --------
-        >>> vfuncs, data = {LR : logistic}, {train_1 : [X1,y1], train2 : [X2,y2]}
-        {(train_1, LR) : fitted logistic, (train_2, LR) :  fitted logistic}
-        """
-        if out_dict is None:
-            out_dict = deepcopy(self.vfuncs)
-
-        apply_func_cached = self._memory.cache(_apply_func_cached)
-        out_dict = apply_func_cached(out_dict, self._async, self._lazy, *args)
-
-        prev = tuple()
-        for arg in args:
-            if PREV_KEY in arg:
-                prev += (arg[PREV_KEY],)
-        out_dict[PREV_KEY] = (self,) + prev
-
-        if self._mlflow is not None:
-            run_dict = {}
-            # log subkeys as params and value as metric
-            for k, v in out_dict.items():
-                if k == PREV_KEY:
-                    continue
-                origins = np.array([subk.origin for subk in k])
-                # ignore init origins and the last origin (this Vset)
-                param_idx = [i for i in range(len(k[:-1])) if origins[i] != "init"]
-                # get or create mlflow run
-                run_dict_key = tuple(subk.value for subk in k[:-1])
-                if run_dict_key in run_dict:
-                    run_id = run_dict[run_dict_key]
-                else:
-                    run = self._mlflow.create_run(self._exp_id)
-                    run_id = run.info.run_id
-                    run_dict[run_dict_key] = run_id
-                    # log params
-                    for idx in param_idx:
-                        subkey = k[idx]
-                        param_name = subkey.origin
-                        # check if the origin occurs multiple times
-                        if np.sum(origins == param_name) > 1:
-                            occurence = np.sum(origins[:idx] == param_name)
-                            param_name = param_name + str(occurence)
-                            self._mlflow.log_param(run_id, param_name, subkey.value)
-                self._mlflow.log_metric(run_id, k[-1].value, v)
-        return out_dict
-
-    def fit(self, *args):
-        """Fits to args using `_apply_func`"""
-        out_dict = {}
-        for k, v in self.vfuncs.items():
-            out_dict[k] = v.fit
-        self.fitted_vfuncs = self._apply_func(*args, out_dict=out_dict)
-        prev = self.fitted_vfuncs[PREV_KEY][1:]
-        if hasattr(self, FILTER_PREV_KEY):
-            prev = getattr(self, FILTER_PREV_KEY) + prev
-        setattr(self, PREV_KEY, prev)
-        self._fitted = True
-        return self
-
-    def fit_transform(self, *args):
-        """Fits to args and transforms only the first arg."""
-        return self.fit(*args).transform(args[0])
-
-    def transform(self, *args):
-        """Transforms args using `_apply_func`"""
-        if not self._fitted:
-            raise AttributeError(
-                "Please fit the Vset object before calling the transform method."
-            )
-        out_dict = {}
-        for k, v in self.fitted_vfuncs.items():
-            if hasattr(v, "transform"):
-                out_dict[k] = v.transform
-        return self._apply_func(*args, out_dict=out_dict)
-
-    def predict(self, *args, with_uncertainty: bool = False, group_by: list = None):
-        """Predicts args using `_apply_func`"""
-        if not self._fitted:
-            raise AttributeError("Please fit the Vset object before calling predict.")
-        pred_dict = {}
-        for k, v in self.fitted_vfuncs.items():
-            if hasattr(v, "predict"):
-                pred_dict[k] = v.predict
-        preds = self._apply_func(*args, out_dict=pred_dict)
-        if with_uncertainty:
-            return prediction_uncertainty(preds, group_by)
-        return preds
-
-    def predict_proba(
-        self, *args, with_uncertainty: bool = False, group_by: list = None
-    ):
-        """Calls predict_proba on args using `_apply_func`"""
-        if not self._fitted:
-            raise AttributeError(
-                "Please fit the Vset object before calling predict_proba."
-            )
-        pred_dict = {}
-        for k, v in self.fitted_vfuncs.items():
-            if hasattr(v, "predict_proba"):
-                pred_dict[k] = v.predict_proba
-        preds = self._apply_func(*args, out_dict=pred_dict)
-        if with_uncertainty:
-            return prediction_uncertainty(preds, group_by)
-        return preds
-
-    def evaluate(self, *args):
-        """Combines dicts before calling `_apply_func`"""
-        return self._apply_func(*args)
-
-    def __call__(self, *args, n_out: int = None, keys=None, **kwargs):
-        """Call args using `_apply_func`, optionally seperating
-        output dictionary into `n_out` dictionaries with `keys`
-        """
-        if keys is None:
-            keys = []
-        if n_out is None:
-            n_out = len(args)
-        out_dict = self._apply_func(*args)
-        if n_out == 1:
-            return out_dict
-        out_dicts = sep_dicts(out_dict, n_out=n_out, keys=keys)
-        # add back prev
-        prev = out_dict[PREV_KEY]
-        for i in range(n_out):
-            if n_out == len(args):
-                out_dicts[i][PREV_KEY] = (prev[0],) + (prev[i + 1],)
-            else:
-                out_dicts[i][PREV_KEY] = prev
-        return out_dicts
-
-    def __getitem__(self, i):
-        """Accesses ith item in the vfunc set"""
-        return self.vfuncs[i]
-
-    def __contains__(self, key):
-        """Returns true if vfuncs is a dict and key is one of its keys"""
-        if isinstance(self.vfuncs, dict):
-            return key in self.vfuncs.keys()
-        return False
-
-    def keys(self):
-        """Returns Vset vfunc keys"""
-        if isinstance(self.vfuncs, dict):
-            return self.vfuncs.keys()
-        return {}.keys()
-
-    def __len__(self):
-        return len(self.vfuncs)
-
-    def __str__(self):
-        return "Vset(" + self.name + ")"
-
-    def __create_subkey(self, value):
-        """Helper function to construct `Subkey` with
-        this Vset determining origin and output_matching
-        """
-        return Subkey(value, self.name, self._output_matching)
    -
    -

    Methods

    -
    -
    -def evaluate(self, *args) -
    -
    -

    Combines dicts before calling _apply_func

    -
    - -Expand source code - -
    def evaluate(self, *args):
-    """Combines dicts before calling `_apply_func`"""
-    return self._apply_func(*args)
    -
    -
    -
    -def fit(self, *args) -
    -
    -

    Fits to args using _apply_func

    -
    - -Expand source code - -
    def fit(self, *args):
-    """Fits to args using `_apply_func`"""
-    out_dict = {}
-    for k, v in self.vfuncs.items():
-        out_dict[k] = v.fit
-    self.fitted_vfuncs = self._apply_func(*args, out_dict=out_dict)
-    prev = self.fitted_vfuncs[PREV_KEY][1:]
-    if hasattr(self, FILTER_PREV_KEY):
-        prev = getattr(self, FILTER_PREV_KEY) + prev
-    setattr(self, PREV_KEY, prev)
-    self._fitted = True
-    return self
    -
    -
    -
    -def fit_transform(self, *args) -
    -
    -

    Fits to args and transforms only the first arg.

    -
    - -Expand source code - -
    def fit_transform(self, *args):
-    """Fits to args and transforms only the first arg."""
-    return self.fit(*args).transform(args[0])
    -
    -
    -
    -def keys(self) -
    -
    -

    Returns Vset vfunc keys

    -
    - -Expand source code - -
    def keys(self):
-    """Returns Vset vfunc keys"""
-    if isinstance(self.vfuncs, dict):
-        return self.vfuncs.keys()
-    return {}.keys()
    -
    -
    -
    -def predict(self, *args, with_uncertainty: bool = False, group_by: list = None) -
    -
    -

    Predicts args using _apply_func

    -
    - -Expand source code - -
    def predict(self, *args, with_uncertainty: bool = False, group_by: list = None):
-    """Predicts args using `_apply_func`"""
-    if not self._fitted:
-        raise AttributeError("Please fit the Vset object before calling predict.")
-    pred_dict = {}
-    for k, v in self.fitted_vfuncs.items():
-        if hasattr(v, "predict"):
-            pred_dict[k] = v.predict
-    preds = self._apply_func(*args, out_dict=pred_dict)
-    if with_uncertainty:
-        return prediction_uncertainty(preds, group_by)
-    return preds
    -
    -
    -
    -def predict_proba(self, *args, with_uncertainty: bool = False, group_by: list = None) -
    -
    -

    Calls predict_proba on args using _apply_func

    -
    - -Expand source code - -
    def predict_proba(
-    self, *args, with_uncertainty: bool = False, group_by: list = None
-):
-    """Calls predict_proba on args using `_apply_func`"""
-    if not self._fitted:
-        raise AttributeError(
-            "Please fit the Vset object before calling predict_proba."
-        )
-    pred_dict = {}
-    for k, v in self.fitted_vfuncs.items():
-        if hasattr(v, "predict_proba"):
-            pred_dict[k] = v.predict_proba
-    preds = self._apply_func(*args, out_dict=pred_dict)
-    if with_uncertainty:
-        return prediction_uncertainty(preds, group_by)
-    return preds
    -
    -
    -
    -def transform(self, *args) -
    -
    -

    Transforms args using _apply_func

    -
    - -Expand source code - -
    def transform(self, *args):
-    """Transforms args using `_apply_func`"""
-    if not self._fitted:
-        raise AttributeError(
-            "Please fit the Vset object before calling the transform method."
-        )
-    out_dict = {}
-    for k, v in self.fitted_vfuncs.items():
-        if hasattr(v, "transform"):
-            out_dict[k] = v.transform
-    return self._apply_func(*args, out_dict=out_dict)
    -
    -
    -
    -
    -

  • Notebooks

    @@ -2587,7 +219,7 @@

    Methods

    • -

  • API

    +

  • API

    • -

  • Functions

    +

  • Functions

    • -

  • Classes

    +

  • Classes

    diff --git a/docs/pipeline.html b/docs/pipeline.html index 33598d2..6e87df4 100644 --- a/docs/pipeline.html +++ b/docs/pipeline.html @@ -82,16 +82,26 @@ self.memory = joblib.Memory(location=cache_dir) def run(self, *args, **kwargs): - """Runs the pipeline""" + """Runs the pipeline: each step is called on the previous step's outputs + (the first on `args`), and the last step's outputs are returned. + + A step that returns a tuple or list (e.g. one output dict per input) + passes them on as separate positional arguments. `kwargs` go to every + step. + """ run_step_cached = self.memory.cache(_run_step) + outputs = args for i, step in enumerate(self.steps): try: step_name = step.name except AttributeError: step_name = f"Step {i}" print(step_name) - _, fitted_step = run_step_cached(step, *args, **kwargs) + outputs, fitted_step = run_step_cached(step, *outputs, **kwargs) self.steps[i] = fitted_step + if not isinstance(outputs, (tuple, list)): + outputs = (outputs,) + return outputs[0] if len(outputs) == 1 else outputs def __getitem__(self, i): """Accesses ith step of pipeline""" @@ -101,17 +111,20 @@ return len(self.steps) def generate_names(self, as_pandas=True): + """Names for every combination of vfuncs across the steps: a DataFrame + with one column per step, or a list of tuples.""" name_lists = [] - if as_pandas: - for step in self.steps: - name_lists.append([f"{i}_{str(mod)[:8]}" for i, mod in enumerate(step)]) - indexes = list(itertools.product(*name_lists)) - return pd.DataFrame(indexes, columns=[step.name for step in self.steps]) for step in self.steps: + # a Vset is not iterable; its vfuncs are + mods = list(step.vfuncs.values()) + prefix = "" if as_pandas else f"{step.name}_" name_lists.append( - [f"{step.name}_{i}_{str(mod)[:8]}" for i, mod in enumerate(step)] + [f"{prefix}{i}_{str(mod)[:8]}" for i, mod in enumerate(mods)] ) - return list(itertools.product(*name_lists)) + indexes = list(itertools.product(*name_lists)) + if as_pandas: + return pd.DataFrame(indexes, columns=[step.name for step in self.steps]) + return indexes def build_graph(node, draw=True): @@ -353,16 +366,26 @@

    Classes

    self.memory = joblib.Memory(location=cache_dir) def run(self, *args, **kwargs): - """Runs the pipeline""" + """Runs the pipeline: each step is called on the previous step's outputs + (the first on `args`), and the last step's outputs are returned. + + A step that returns a tuple or list (e.g. one output dict per input) + passes them on as separate positional arguments. `kwargs` go to every + step. + """ run_step_cached = self.memory.cache(_run_step) + outputs = args for i, step in enumerate(self.steps): try: step_name = step.name except AttributeError: step_name = f"Step {i}" print(step_name) - _, fitted_step = run_step_cached(step, *args, **kwargs) + outputs, fitted_step = run_step_cached(step, *outputs, **kwargs) self.steps[i] = fitted_step + if not isinstance(outputs, (tuple, list)): + outputs = (outputs,) + return outputs[0] if len(outputs) == 1 else outputs def __getitem__(self, i): """Accesses ith step of pipeline""" @@ -372,17 +395,20 @@

    Classes

    return len(self.steps) def generate_names(self, as_pandas=True): + """Names for every combination of vfuncs across the steps: a DataFrame + with one column per step, or a list of tuples.""" name_lists = [] - if as_pandas: - for step in self.steps: - name_lists.append([f"{i}_{str(mod)[:8]}" for i, mod in enumerate(step)]) - indexes = list(itertools.product(*name_lists)) - return pd.DataFrame(indexes, columns=[step.name for step in self.steps]) for step in self.steps: + # a Vset is not iterable; its vfuncs are + mods = list(step.vfuncs.values()) + prefix = "" if as_pandas else f"{step.name}_" name_lists.append( - [f"{step.name}_{i}_{str(mod)[:8]}" for i, mod in enumerate(step)] + [f"{prefix}{i}_{str(mod)[:8]}" for i, mod in enumerate(mods)] ) - return list(itertools.product(*name_lists))     + indexes = list(itertools.product(*name_lists)) + if as_pandas: + return pd.DataFrame(indexes, columns=[step.name for step in self.steps]) + return indexes

    Methods

    @@ -390,45 +416,63 @@

    Methods

    def generate_names(self, as_pandas=True)
    -
    +

    Names for every combination of vfuncs across the steps: a DataFrame +with one column per step, or a list of tuples.

    Expand source code 
    def generate_names(self, as_pandas=True):
+    """Names for every combination of vfuncs across the steps: a DataFrame
+    with one column per step, or a list of tuples."""
     name_lists = []
-    if as_pandas:
-        for step in self.steps:
-            name_lists.append([f"{i}_{str(mod)[:8]}" for i, mod in enumerate(step)])
-        indexes = list(itertools.product(*name_lists))
-        return pd.DataFrame(indexes, columns=[step.name for step in self.steps])
     for step in self.steps:
+        # a Vset is not iterable; its vfuncs are
+        mods = list(step.vfuncs.values())
+        prefix = "" if as_pandas else f"{step.name}_"
         name_lists.append(
-            [f"{step.name}_{i}_{str(mod)[:8]}" for i, mod in enumerate(step)]
+            [f"{prefix}{i}_{str(mod)[:8]}" for i, mod in enumerate(mods)]
         )
-    return list(itertools.product(*name_lists))
    + indexes = list(itertools.product(*name_lists)) + if as_pandas: + return pd.DataFrame(indexes, columns=[step.name for step in self.steps]) + return indexes
    def run(self, *args, **kwargs)
    -

    Runs the pipeline

    +

    Runs the pipeline: each step is called on the previous step's outputs +(the first on args), and the last step's outputs are returned.

    +

    A step that returns a tuple or list (e.g. one output dict per input) +passes them on as separate positional arguments. kwargs go to every +step.

    Expand source code 
    def run(self, *args, **kwargs):
-    """Runs the pipeline"""
+    """Runs the pipeline: each step is called on the previous step's outputs
+    (the first on `args`), and the last step's outputs are returned.
+
+    A step that returns a tuple or list (e.g. one output dict per input)
+    passes them on as separate positional arguments. `kwargs` go to every
+    step.
+    """
     run_step_cached = self.memory.cache(_run_step)
+    outputs = args
     for i, step in enumerate(self.steps):
         try:
             step_name = step.name
         except AttributeError:
             step_name = f"Step {i}"
         print(step_name)
-        _, fitted_step = run_step_cached(step, *args, **kwargs)
-        self.steps[i] = fitted_step
    + outputs, fitted_step = run_step_cached(step, *outputs, **kwargs) + self.steps[i] = fitted_step + if not isinstance(outputs, (tuple, list)): + outputs = (outputs,) + return outputs[0] if len(outputs) == 1 else outputs
    @@ -442,7 +486,6 @@

    Methods

  • Notebooks

    diff --git a/docs/style_docs.py b/docs/style_docs.py index dcde317..2f4ed30 100644 --- a/docs/style_docs.py +++ b/docs/style_docs.py @@ -1,17 +1,28 @@ """Post-process the pdoc-generated index page. -Run from docs/ after pdoc has written index.html; build_pages.py then builds -the article pages from the result. +Run after pdoc has written the API pages into docs/; build_pages.py then +builds the article pages from the result. """ +import glob import os import re -INDEX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "index.html") +HERE = os.path.dirname(os.path.abspath(__file__)) +INDEX = os.path.join(HERE, "index.html") with open(INDEX, encoding="utf-8") as f: data = f.read() + +def drop(pattern, text, what): + """Remove `pattern` from `text`, and say so when it matches nothing.""" + new_text, n = re.subn(pattern, "", text, flags=re.S) + if n == 0: + print(f" style_docs.py: nothing removed for {what}; has the readme changed?") + return new_text + + # the readme opens with a large centered logo, which the header bar now carries data = re.sub( r'

    \s*]*>\s*

    ', @@ -23,6 +34,52 @@ # a self-referential "Docs" link from the readme is noise on the docs site itself data = data.replace('Docs', "") +# The index is the readme, which ends with References. Everything pdoc appends +# after it (the sub-module list and the re-exported functions and classes) is +# documented on the module pages, so cut it from the article. +data = drop( + r'
    \s*

    .*?(?=)', + data, + "the API sections after References", +) + +# pdoc's "Expand source code" toggle for the package __init__ would be the last +# thing on the page, after References +data = drop( + r'
    .*?
    \s*(?=

    )', + data, + "the package source toggle", +) + +# The sidebar section headings pointed at those sections; keep them as plain +# labels. +data = re.sub(r'

    (.*?)

    ', r"

    \1

    ", data) + +# The sidebar's function and class links pointed at anchors on this page that +# no longer exist. Point each at the module page that documents it: +# "#vflow.build_vset" -> "helpers.html#vflow.helpers.build_vset". +anchors = {} +for page in glob.glob(os.path.join(HERE, "*.html")): + name = os.path.basename(page) + if name == "index.html": + continue + with open(page, encoding="utf-8") as f: + for ref in re.findall(r'
    ', f.read()): + # vflow.. -> + anchors.setdefault(ref.split(".", 2)[-1], f"{name}#{ref}") + + +def _repoint(match): + target = anchors.get(match.group(1)) + return f'href="{target}"' if target else match.group(0) + + +data, n_repointed = re.subn(r'href="#vflow\.([\w.]+)"', _repoint, data) +left = sorted(set(re.findall(r'href="#vflow\.([\w.]+)"', data))) +print(f" style_docs.py: repointed {n_repointed} sidebar links to module pages") +if left: + print(f" style_docs.py: no module page documents {left}") + with open(INDEX, "w", encoding="utf-8") as f: f.write(data) print(" style_docs.py: cleaned index.html") diff --git a/docs/subkey.html b/docs/subkey.html index af09e80..3aced68 100644 --- a/docs/subkey.html +++ b/docs/subkey.html @@ -134,8 +134,9 @@ return str(self.value) def __hash__(self): - """Mainly used for testing purposes.""" - return hash(self.value) ^ hash(self.origin) ^ hash(self.output_matching) + """Consistent with __eq__, which compares value and origin only: keys + that compare equal must hash equal or dict and set lookups miss them.""" + return hash(self.value) ^ hash(self.origin)
    @@ -243,8 +244,9 @@

    Classes

    return str(self.value) def __hash__(self): - """Mainly used for testing purposes.""" - return hash(self.value) ^ hash(self.origin) ^ hash(self.output_matching) + """Consistent with __eq__, which compares value and origin only: keys + that compare equal must hash equal or dict and set lookups miss them.""" + return hash(self.value) ^ hash(self.origin)

    Methods

    @@ -355,7 +357,6 @@

    Methods

  • Notebooks

    diff --git a/docs/utils.html b/docs/utils.html index 7e0be66..5e05edb 100644 --- a/docs/utils.html +++ b/docs/utils.html @@ -283,20 +283,26 @@ df = df.join(param_df) else: param_loc = df.columns.get_loc(param_key) - param_key_cols = [ - f"{p.split('=')[0]}-{param_key}" for p in param_keys[0] - ] - param_keys = [[s.split("=")[1] for s in t] for t in param_keys] - df = df.join(pd.DataFrame(param_keys)).drop(columns=param_key) - new_cols = df.columns[: len(cols) - 1].tolist() + param_key_cols - df = df.set_axis(new_cols, axis=1) - new_idx = list(range(len(new_cols))) - new_idx = ( - new_idx[:param_loc] - + new_idx[len(cols) - 1 :] - + new_idx[param_loc : len(cols) - 1] + # parse every row on its own: vfuncs built from different funcs + # can have different parameter names + rows = [] + for t in param_keys: + if isinstance(t, str) or not all( + isinstance(s, str) and "=" in s for s in t + ): + raise ValueError( + f"param_key={param_key!r} needs keys that are tuples " + "of 'name=value' strings, as build_vset(param_dict=...) " + f"makes; got {t!r}" + ) + # split on the first '=' only, so values may contain '=' + rows.append(dict(s.split("=", 1) for s in t)) + param_df = pd.DataFrame(rows, index=df.index) + param_df.columns = [f"{c}-{param_key}" for c in param_df.columns] + df = pd.concat( + [df.iloc[:, :param_loc], param_df, df.iloc[:, param_loc + 1 :]], + axis=1, ) - df = df.iloc[:, new_idx] return df @@ -340,6 +346,11 @@ """ if func is None: func = ["count", "mean", "std"] + if isinstance(func, str): + func = [func] + # for scalar outputs, columns are prefixed only when a prefix is asked for, + # so existing callers keep their column names + prefix_scalar_cols = prefix is not None if prefix is None: prefix = wrt if isinstance(data, dict): @@ -355,7 +366,7 @@ gb = df.groupby(group_by)[wrt] else: gb = df.groupby(lambda x: True)[wrt] - if (isinstance(func, list) and "mean" in func or "std" in func) and ( + if (isinstance(func, list) and ("mean" in func or "std" in func)) and ( type(df[wrt].iloc[0]) in [list, np.ndarray] ): # iterate the groups (same order as gb.groups); get_group with a single @@ -393,9 +404,13 @@ wrt_stds.index.names = df_out.index.names df_out = df_out.join(wrt_stds) if "count" not in func: - df_out = df_out.drop(f"{prefix}-count") + df_out = df_out.drop(columns=f"{prefix}-count") else: df_out = gb.agg(func) + if isinstance(df_out, pd.Series): # a single callable gives a Series + df_out = df_out.to_frame(getattr(func, "__name__", wrt)) + if prefix_scalar_cols: + df_out.columns = [f"{prefix}-{c}" for c in df_out.columns] df_out = df_out.reindex(sorted(df_out.columns), axis=1) df_out.reset_index(inplace=True) if len(group_by) > 0: @@ -900,20 +915,26 @@

    Returns

    df = df.join(param_df) else: param_loc = df.columns.get_loc(param_key) - param_key_cols = [ - f"{p.split('=')[0]}-{param_key}" for p in param_keys[0] - ] - param_keys = [[s.split("=")[1] for s in t] for t in param_keys] - df = df.join(pd.DataFrame(param_keys)).drop(columns=param_key) - new_cols = df.columns[: len(cols) - 1].tolist() + param_key_cols - df = df.set_axis(new_cols, axis=1) - new_idx = list(range(len(new_cols))) - new_idx = ( - new_idx[:param_loc] - + new_idx[len(cols) - 1 :] - + new_idx[param_loc : len(cols) - 1] + # parse every row on its own: vfuncs built from different funcs + # can have different parameter names + rows = [] + for t in param_keys: + if isinstance(t, str) or not all( + isinstance(s, str) and "=" in s for s in t + ): + raise ValueError( + f"param_key={param_key!r} needs keys that are tuples " + "of 'name=value' strings, as build_vset(param_dict=...) " + f"makes; got {t!r}" + ) + # split on the first '=' only, so values may contain '=' + rows.append(dict(s.split("=", 1) for s in t)) + param_df = pd.DataFrame(rows, index=df.index) + param_df.columns = [f"{c}-{param_key}" for c in param_df.columns] + df = pd.concat( + [df.iloc[:, :param_loc], param_df, df.iloc[:, param_loc + 1 :]], + axis=1, ) - df = df.iloc[:, new_idx] return df @@ -1028,6 +1049,11 @@

    Returns

    """ if func is None: func = ["count", "mean", "std"] + if isinstance(func, str): + func = [func] + # for scalar outputs, columns are prefixed only when a prefix is asked for, + # so existing callers keep their column names + prefix_scalar_cols = prefix is not None if prefix is None: prefix = wrt if isinstance(data, dict): @@ -1043,7 +1069,7 @@

    Returns

    gb = df.groupby(group_by)[wrt] else: gb = df.groupby(lambda x: True)[wrt] - if (isinstance(func, list) and "mean" in func or "std" in func) and ( + if (isinstance(func, list) and ("mean" in func or "std" in func)) and ( type(df[wrt].iloc[0]) in [list, np.ndarray] ): # iterate the groups (same order as gb.groups); get_group with a single @@ -1081,9 +1107,13 @@

    Returns

    wrt_stds.index.names = df_out.index.names df_out = df_out.join(wrt_stds) if "count" not in func: - df_out = df_out.drop(f"{prefix}-count") + df_out = df_out.drop(columns=f"{prefix}-count") else: df_out = gb.agg(func) + if isinstance(df_out, pd.Series): # a single callable gives a Series + df_out = df_out.to_frame(getattr(func, "__name__", wrt)) + if prefix_scalar_cols: + df_out.columns = [f"{prefix}-{c}" for c in df_out.columns] df_out = df_out.reindex(sorted(df_out.columns), axis=1) df_out.reset_index(inplace=True) if len(group_by) > 0: @@ -1330,7 +1360,6 @@

    Examples

  • Notebooks

    diff --git a/docs/vflow-internals.html b/docs/vflow-internals.html index c7c8238..e3c8f6e 100644 --- a/docs/vflow-internals.html +++ b/docs/vflow-internals.html @@ -166,7 +166,6 @@

    Behavior when Subkey has output_matching=True

  • Notebooks

    @@ -195,7 +194,7 @@

    Behavior when Subkey has output_matching=True

    • -

  • API

    +

  • API

    • -

  • Functions

    +

  • Functions

    • -

  • Classes

    +

  • Classes

    diff --git a/docs/vfunc.html b/docs/vfunc.html index e9116e6..e11f4bc 100644 --- a/docs/vfunc.html +++ b/docs/vfunc.html @@ -143,6 +143,12 @@ tmp_args[i] = tmp_args[i]() while isinstance(self.vfunc, VfuncPromise): self.vfunc = self.vfunc() + # an upstream lazy async Vset leaves Ray references; a remote call + # resolves them itself, but a plain function must get the values + if not isinstance(getattr(self.vfunc, "__self__", None), AsyncVfunc): + tmp_args = [ + ray.get(a) if isinstance(a, ray.ObjectRef) else a for a in tmp_args + ] self.value = self.vfunc(*tmp_args) self.called = True return self.value @@ -361,6 +367,12 @@

    Methods

    tmp_args[i] = tmp_args[i]() while isinstance(self.vfunc, VfuncPromise): self.vfunc = self.vfunc() + # an upstream lazy async Vset leaves Ray references; a remote call + # resolves them itself, but a plain function must get the values + if not isinstance(getattr(self.vfunc, "__self__", None), AsyncVfunc): + tmp_args = [ + ray.get(a) if isinstance(a, ray.ObjectRef) else a for a in tmp_args + ] self.value = self.vfunc(*tmp_args) self.called = True return self.value @@ -442,7 +454,6 @@

    Methods

  • Notebooks

    diff --git a/docs/vset.html b/docs/vset.html index 00a19ec..42e78f5 100644 --- a/docs/vset.html +++ b/docs/vset.html @@ -57,7 +57,9 @@ Function arguments are each a list """ +import hashlib import os +import warnings from copy import deepcopy import joblib @@ -65,6 +67,11 @@ import ray from mlflow.tracking import MlflowClient +try: + import cloudpickle +except ImportError: # ray always ships a copy + from ray import cloudpickle + from vflow.subkey import Subkey from vflow.utils import ( PREV_KEY, @@ -122,7 +129,8 @@ self._output_matching = output_matching self._lazy = lazy self._cache_dir = cache_dir - self._memory = joblib.Memory(self._cache_dir) + # verbose=0: by default joblib prints every call with its full arguments + self._memory = joblib.Memory(self._cache_dir, verbose=0) if tracking_dir is not None: if "://" not in str(tracking_dir): # a plain directory: MLflow 3 needs an explicit opt-in for its @@ -136,13 +144,27 @@ self._exp_id = experiment.experiment_id else: self._mlflow = None + if not isinstance(vfuncs, (dict, list)): + raise TypeError( + f"vfuncs must be a list or a dict, not {type(vfuncs).__name__}" + ) # check if any of the vfuncs are AsyncVfuncs # if so, we'll make then all AsyncVfuncs later on - if not self._async and np.any([isinstance(vf, AsyncVfunc) for vf in vfuncs]): + funcs = vfuncs.values() if isinstance(vfuncs, dict) else vfuncs + if not self._async and any(isinstance(vf, AsyncVfunc) for vf in funcs): self._async = True if isinstance(vfuncs, dict): - self.vfuncs = vfuncs - elif isinstance(vfuncs, list): + # keys may already be tuples of Subkeys (e.g. from another Vset); + # plain keys like "LR" become this Vset's subkeys + self.vfuncs = { + ( + k + if isinstance(k, tuple) and all(isinstance(s, Subkey) for s in k) + else (self.__create_subkey(k),) + ): v + for k, v in vfuncs.items() + } + else: if vfunc_keys is not None: assert isinstance( vfunc_keys, list @@ -150,6 +172,11 @@ assert len(vfuncs) == len( vfunc_keys ), "vfuncs list and vfunc_keys list do not have the same length" + if len(set(map(repr, vfunc_keys))) < len(vfunc_keys): + raise ValueError( + f"vfunc_keys must be unique; duplicates would silently " + f"drop vfuncs: {vfunc_keys}" + ) # TODO: how best to handle tuple subkeys? vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] else: @@ -194,8 +221,29 @@ if out_dict is None: out_dict = deepcopy(self.vfuncs) - apply_func_cached = self._memory.cache(_apply_func_cached) - out_dict = apply_func_cached(out_dict, self._async, self._lazy, *args) + cache_key = None + if self._cache_dir is not None: + # joblib hashes functions by name, so redefining a function would + # serve stale results, and it cannot hash lambdas at all. Key the + # cache on the functions' code and the inputs instead. + try: + cache_key = hashlib.sha256( + cloudpickle.dumps((out_dict, self._async, self._lazy, args)) + ).hexdigest() + except Exception as e: # noqa: BLE001 - fall back to no caching + warnings.warn( + f"Vset {self.name!r}: could not fingerprint the inputs for " + f"caching ({type(e).__name__}: {e}); running without the cache" + ) + if cache_key is None: + out_dict = _apply_func_cached(None, out_dict, self._async, self._lazy, args) + else: + apply_func_cached = self._memory.cache( + _apply_func_cached, ignore=["out_dict", "args"] + ) + out_dict = apply_func_cached( + cache_key, out_dict, self._async, self._lazy, args + ) prev = tuple() for arg in args: @@ -207,7 +255,9 @@ run_dict = {} # log subkeys as params and value as metric for k, v in out_dict.items(): - if k == PREV_KEY: + # only numeric scalars are metrics; fitted models, predictions + # and lazy promises are not logged + if k == PREV_KEY or not _is_numeric_scalar(v): continue origins = np.array([subk.origin for subk in k]) # ignore init origins and the last origin (this Vset) @@ -228,8 +278,11 @@ if np.sum(origins == param_name) > 1: occurence = np.sum(origins[:idx] == param_name) param_name = param_name + str(occurence) - self._mlflow.log_param(run_id, param_name, subkey.value) - self._mlflow.log_metric(run_id, k[-1].value, v) + self._mlflow.log_param(run_id, param_name, subkey.value) + self._mlflow.log_metric(run_id, k[-1].value, float(v)) + # close the runs, or they stay RUNNING forever + for run_id in run_dict.values(): + self._mlflow.set_terminated(run_id) return out_dict def fit(self, *args): @@ -269,6 +322,7 @@ for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict"): pred_dict[k] = v.predict + self._require_method(pred_dict, "predict") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -286,6 +340,7 @@ for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict_proba"): pred_dict[k] = v.predict_proba + self._require_method(pred_dict, "predict_proba") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -338,6 +393,15 @@ def __str__(self): return "Vset(" + self.name + ")" + def _require_method(self, method_dict, method): + """Raise rather than silently return no outputs when no fitted vfunc + has `method`.""" + if not method_dict: + raise AttributeError( + f"None of the fitted vfuncs in Vset {self.name!r} has a " + f"{method!r} method." + ) + def __create_subkey(self, value): """Helper function to construct `Subkey` with this Vset determining origin and output_matching @@ -345,19 +409,31 @@ return Subkey(value, self.name, self._output_matching) -def _apply_func_cached(out_dict: dict, is_async: bool, lazy: bool, *args): +def _is_numeric_scalar(v) -> bool: + """Whether `v` can be logged as an MLflow metric.""" + return isinstance(v, (int, float, np.integer, np.floating)) and not isinstance( + v, bool + ) + + +def _apply_func_cached( + cache_key, out_dict: dict, is_async: bool, lazy: bool, args: tuple +): """ Params ------ - *args: dict - Takes multiple dicts and combines them into one. - Then runs vfuncs on each item in combined dict. + cache_key: str or None + Fingerprint of the vfuncs and inputs; the only argument joblib hashes + when caching (out_dict and args are ignored by the cache). out_dict: dict The dictionary to pass to the matching function. is_async: bool If True, outputs are computed asynchronously. lazy: bool If True, outputs are evaluated lazily, i.e. outputs are `VfuncPromise`. + args: tuple of dict + Takes multiple dicts and combines them into one. + Then runs vfuncs on each item in combined dict. Returns ------- @@ -553,7 +629,8 @@

    Classes

    self._output_matching = output_matching self._lazy = lazy self._cache_dir = cache_dir - self._memory = joblib.Memory(self._cache_dir) + # verbose=0: by default joblib prints every call with its full arguments + self._memory = joblib.Memory(self._cache_dir, verbose=0) if tracking_dir is not None: if "://" not in str(tracking_dir): # a plain directory: MLflow 3 needs an explicit opt-in for its @@ -567,13 +644,27 @@

    Classes

    self._exp_id = experiment.experiment_id else: self._mlflow = None + if not isinstance(vfuncs, (dict, list)): + raise TypeError( + f"vfuncs must be a list or a dict, not {type(vfuncs).__name__}" + ) # check if any of the vfuncs are AsyncVfuncs # if so, we'll make then all AsyncVfuncs later on - if not self._async and np.any([isinstance(vf, AsyncVfunc) for vf in vfuncs]): + funcs = vfuncs.values() if isinstance(vfuncs, dict) else vfuncs + if not self._async and any(isinstance(vf, AsyncVfunc) for vf in funcs): self._async = True if isinstance(vfuncs, dict): - self.vfuncs = vfuncs - elif isinstance(vfuncs, list): + # keys may already be tuples of Subkeys (e.g. from another Vset); + # plain keys like "LR" become this Vset's subkeys + self.vfuncs = { + ( + k + if isinstance(k, tuple) and all(isinstance(s, Subkey) for s in k) + else (self.__create_subkey(k),) + ): v + for k, v in vfuncs.items() + } + else: if vfunc_keys is not None: assert isinstance( vfunc_keys, list @@ -581,6 +672,11 @@

    Classes

    assert len(vfuncs) == len( vfunc_keys ), "vfuncs list and vfunc_keys list do not have the same length" + if len(set(map(repr, vfunc_keys))) < len(vfunc_keys): + raise ValueError( + f"vfunc_keys must be unique; duplicates would silently " + f"drop vfuncs: {vfunc_keys}" + ) # TODO: how best to handle tuple subkeys? vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] else: @@ -625,8 +721,29 @@

    Classes

    if out_dict is None: out_dict = deepcopy(self.vfuncs) - apply_func_cached = self._memory.cache(_apply_func_cached) - out_dict = apply_func_cached(out_dict, self._async, self._lazy, *args) + cache_key = None + if self._cache_dir is not None: + # joblib hashes functions by name, so redefining a function would + # serve stale results, and it cannot hash lambdas at all. Key the + # cache on the functions' code and the inputs instead. + try: + cache_key = hashlib.sha256( + cloudpickle.dumps((out_dict, self._async, self._lazy, args)) + ).hexdigest() + except Exception as e: # noqa: BLE001 - fall back to no caching + warnings.warn( + f"Vset {self.name!r}: could not fingerprint the inputs for " + f"caching ({type(e).__name__}: {e}); running without the cache" + ) + if cache_key is None: + out_dict = _apply_func_cached(None, out_dict, self._async, self._lazy, args) + else: + apply_func_cached = self._memory.cache( + _apply_func_cached, ignore=["out_dict", "args"] + ) + out_dict = apply_func_cached( + cache_key, out_dict, self._async, self._lazy, args + ) prev = tuple() for arg in args: @@ -638,7 +755,9 @@

    Classes

    run_dict = {} # log subkeys as params and value as metric for k, v in out_dict.items(): - if k == PREV_KEY: + # only numeric scalars are metrics; fitted models, predictions + # and lazy promises are not logged + if k == PREV_KEY or not _is_numeric_scalar(v): continue origins = np.array([subk.origin for subk in k]) # ignore init origins and the last origin (this Vset) @@ -659,8 +778,11 @@

    Classes

    if np.sum(origins == param_name) > 1: occurence = np.sum(origins[:idx] == param_name) param_name = param_name + str(occurence) - self._mlflow.log_param(run_id, param_name, subkey.value) - self._mlflow.log_metric(run_id, k[-1].value, v) + self._mlflow.log_param(run_id, param_name, subkey.value) + self._mlflow.log_metric(run_id, k[-1].value, float(v)) + # close the runs, or they stay RUNNING forever + for run_id in run_dict.values(): + self._mlflow.set_terminated(run_id) return out_dict def fit(self, *args): @@ -700,6 +822,7 @@

    Classes

    for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict"): pred_dict[k] = v.predict + self._require_method(pred_dict, "predict") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -717,6 +840,7 @@

    Classes

    for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict_proba"): pred_dict[k] = v.predict_proba + self._require_method(pred_dict, "predict_proba") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -769,6 +893,15 @@

    Classes

    def __str__(self): return "Vset(" + self.name + ")" + def _require_method(self, method_dict, method): + """Raise rather than silently return no outputs when no fitted vfunc + has `method`.""" + if not method_dict: + raise AttributeError( + f"None of the fitted vfuncs in Vset {self.name!r} has a " + f"{method!r} method." + ) + def __create_subkey(self, value): """Helper function to construct `Subkey` with this Vset determining origin and output_matching @@ -861,6 +994,7 @@

    Methods

    for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict"): pred_dict[k] = v.predict + self._require_method(pred_dict, "predict") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -888,6 +1022,7 @@

    Methods

    for k, v in self.fitted_vfuncs.items(): if hasattr(v, "predict_proba"): pred_dict[k] = v.predict_proba + self._require_method(pred_dict, "predict_proba") preds = self._apply_func(*args, out_dict=pred_dict) if with_uncertainty: return prediction_uncertainty(preds, group_by) @@ -927,7 +1062,6 @@

    Methods

  • Notebooks

    From 2091bdbb9bc084af80c52df7970e170b90d4192f Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Thu, 15 Oct 2026 17:07:14 -0400 Subject: [PATCH 4/6] Fix bugs found in a second review round - caching: key the cache on a by-value hash of the inputs (joblib.hash, without the lineage entry) so equal copies, e.g. outputs loaded back from the cache, hit it; add the source of module-level functions so editing one does not serve stale results - MLflow: metric names from build_vset tuple keys are sanitized; runs are closed even if logging fails; runs are keyed by full subkeys so equal values from different Vsets are not merged - PCSPipeline.run passed Vfunc objects on from an already-fitted step - vfunc_keys that compare equal (1 and 1.0) were not caught as duplicates - perturbation_stats on array entries: tuple group keys (build_vset models, predict_proba with uncertainty grouped by model) crashed, split=True crashed on 2-D entries (now out_ columns), and unsupported funcs were silently ignored (now raise) - build_vset: partials of one function get distinct names, and distinct functions or values that share a key raise instead of being dropped Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Xy8Dg7fAqocHGzMqVCiSK --- tests/test_bugfixes.py | 128 +++++++++++++++++++++++++++++++++++++ tests/test_seeding.py | 2 +- vflow/helpers.py | 45 +++++++++++-- vflow/pipeline.py | 8 ++- vflow/utils.py | 59 ++++++++--------- vflow/vset.py | 139 ++++++++++++++++++++++++++++++----------- 6 files changed, 308 insertions(+), 73 deletions(-) diff --git a/tests/test_bugfixes.py b/tests/test_bugfixes.py index 96cf223..cae8ec1 100644 --- a/tests/test_bugfixes.py +++ b/tests/test_bugfixes.py @@ -269,3 +269,131 @@ def test_async_detected_from_dict_of_async_vfuncs(): vset = Vset("s", {(Subkey("s", "s"),): AsyncVfunc("s", np.sum)}) assert vset._async assert values(vset(x)) == [3.0] + + +class TestArrayPerturbationStats: + def setup_method(self): + # two 3x2 "predict_proba" outputs in one group + self.df = pd.DataFrame( + {"g": ["a", "a"], "out": [np.zeros((3, 2)), np.ones((3, 2))]} + ) + + def test_split_flattens_2d_entries(self): + out = perturbation_stats(self.df, "g", split=True) + assert "out2_1-mean" in out.columns and "out0_0-std" in out.columns + assert out["out2_1-mean"].iloc[0] == 0.5 + + def test_tuple_group_keys(self): + df = self.df.assign(g=[("f", "C=1"), ("f", "C=1")]) + out = perturbation_stats(df, "g") + assert len(out) == 1 and out["out-mean"].iloc[0].shape == (3, 2) + + def test_predict_proba_with_uncertainty_grouped_by_model(self): + from sklearn.linear_model import LogisticRegression + + rng = np.random.default_rng(0) + X, y = init_args([rng.normal(size=(20, 2)), [0, 1] * 10], names=["X", "y"]) + boot = build_vset("boot", sklearn_resample, reps=2, seed=0) + Xb, yb = boot(X, y) + models = build_vset("model", LogisticRegression, param_dict={"C": [0.1, 1]}) + models.fit(Xb, yb) + mean, std, _ = models.predict_proba( + X, with_uncertainty=True, group_by=["model"] + ) + assert len(values(mean)) == 2 + + def test_unsupported_funcs_raise(self): + with pytest.raises(ValueError, match="supports only"): + perturbation_stats(self.df, "g", func=["mean", "max"]) + + +def sklearn_resample(*arrays, random_state=None): + from sklearn.utils import resample + + return resample(*arrays, random_state=random_state) + + +class TestBuildVsetNames: + def test_partials_of_one_function_are_kept_apart(self): + from functools import partial + + def draw(X, n=1, random_state=None): + return n + + vset = build_vset( + "d", [partial(draw, n=5), partial(draw, n=10)], reps=2, seed=0 + ) + assert len(vset) == 4 + + def test_distinct_configs_with_one_key_raise(self): + with pytest.raises(ValueError, match="share the key"): + build_vset( + "m", [lambda x, a: x + a, lambda x, a: x * a], param_dict={"a": [1]} + ) + with pytest.raises(ValueError, match="share the key"): + build_vset("n", model, param_dict={"a": [1, "1"]}) + + +class TestCacheKeys: + def test_equal_copy_of_inputs_hits_the_cache(self, tmp_path): + import cloudpickle + + log = str(tmp_path / "calls.txt") + + def stage2(a): + with open(log, "a") as f: + f.write("x") + return a * 2 + + (x,) = init_args([np.array([1.0, 2.0])], names=["x"]) + out1 = Vset("s1", [lambda a: a + 1], vfunc_keys=["p"])(x) + v2 = Vset("s2", [stage2], vfunc_keys=["d"], cache_dir=str(tmp_path / "c")) + v2(out1) + # an output loaded back from a cache is an equal copy, not the object + v2(cloudpickle.loads(cloudpickle.dumps(out1))) + with open(log) as f: + assert f.read() == "x" + + def test_edited_module_function_is_not_served_stale(self, tmp_path, monkeypatch): + import importlib + + mod = tmp_path / "vflow_test_mod.py" + mod.write_text("def f(x):\n return float(x.sum())\n") + monkeypatch.syspath_prepend(str(tmp_path)) + m = importlib.import_module("vflow_test_mod") + (x,) = init_args([np.array([1.0, 2.0])], names=["x"]) + cache = str(tmp_path / "c") + assert values(Vset("s", [m.f], vfunc_keys=["f"], cache_dir=cache)(x)) == [3.0] + mod.write_text("def f(x):\n return float(x.sum()) * 100\n") + m = importlib.reload(m) + out = values(Vset("s", [m.f], vfunc_keys=["f"], cache_dir=cache)(x)) + assert out == [300.0] + + +def test_mlflow_metric_names_from_build_vset_keys(tmp_path): + from mlflow.tracking import MlflowClient + + def score(x, scale=1): + return float(np.sum(x)) * scale + + (x,) = init_args([np.array([1.0, 2.0])], names=["x"]) + met = build_vset( + "met", score, param_dict={"scale": [1, 2]}, tracking_dir=str(tmp_path) + ) + met(x) + (run,) = MlflowClient(tracking_uri=str(tmp_path)).search_runs([met._exp_id]) + assert run.data.metrics == {"func_score_scale_1": 3.0, "func_score_scale_2": 6.0} + assert run.info.status == "FINISHED" + + +def test_pipeline_run_with_a_fitted_step(): + a = Vset("a", [np.abs], vfunc_keys=["abs"]) + b = Vset("b", [np.sum], vfunc_keys=["sum"]) + (x,) = init_args([np.array([-1.0, 2.0])], names=["x"]) + a.fit(x) + assert values(PCSPipeline([a, b]).run(x)) == [3.0] + + +def test_vfunc_keys_that_compare_equal_are_duplicates(): + with pytest.raises(ValueError, match="unique"): + Vset("m", [np.sum, np.mean], vfunc_keys=[1, 1.0]) diff --git a/tests/test_seeding.py b/tests/test_seeding.py index 1b55842..e522ee2 100644 --- a/tests/test_seeding.py +++ b/tests/test_seeding.py @@ -75,7 +75,7 @@ def test_partial_and_callable_object_names(self): vset = build_vset( "boot", partial(resample, replace=True), param_dict={"n_samples": [5]} ) - assert next(iter(vset.vfuncs))[0].value[0] == "func=resample" + assert next(iter(vset.vfuncs))[0].value[0] == "func=resample(replace=True)" class Scale: def __call__(self, X, factor=1): diff --git a/vflow/helpers.py b/vflow/helpers.py index 5d0e37b..5edf227 100644 --- a/vflow/helpers.py +++ b/vflow/helpers.py @@ -48,13 +48,34 @@ def _key_origin(vset: Vset) -> str: def _func_name(f) -> str: - """A readable name for `f`: a function's __name__, the wrapped function's - for a functools.partial, the class name for a callable object.""" + """A readable name for `f`: a function's __name__, the wrapped function + and its bound arguments for a functools.partial (so partials of one + function get distinct names), the class name for a callable object.""" if isinstance(f, partial): - return _func_name(f.func) + bound = [repr(a) for a in f.args] + bound += [f"{k}={v!r}" for k, v in f.keywords.items()] + return f"{_func_name(f.func)}({', '.join(bound)})" return getattr(f, "__name__", None) or type(f).__name__ +def _same_config(a, b) -> bool: + """Whether two (func, param values) pairs from build_vset are the same.""" + (fa, va), (fb, vb) = a, b + if fa is not fb or len(va) != len(vb): + return False + for x, y in zip(va, vb): + if x is y: + continue + if type(x) is not type(y): + return False + try: + if not bool(x == y): + return False + except (TypeError, ValueError): # e.g. arrays, whose == is elementwise + return False + return True + + def _accepts_random_state(f) -> bool: """Whether `f` (a function, partial, class or callable object) can take a `random_state` keyword argument. Returns True when it cannot be told.""" @@ -158,6 +179,9 @@ def build_vset( vfuncs = [] vkeys = [] + # (func, param values) behind each key, to tell a repeated configuration + # from distinct ones that happen to print the same + configs = [] if seed is not None: if "random_state" in kwargs or any( @@ -202,6 +226,7 @@ def build_vset( vkeys.append((f"rep={i}",) + vkey_tup) else: vkeys.append(vkey_tup) + configs.append((f, tup)) rep_kwargs = dict(kwargs_dict) if seed is not None: rep_kwargs["random_state"] = next(seeds) @@ -218,7 +243,19 @@ def build_vset( vkeys = None elif len(set(vkeys)) < len(vkeys): # overlapping param_dicts repeat a configuration; keep one vfunc per - # key (the last, as a dict would), in first-seen order + # key (the last, as a dict would), in first-seen order. Distinct + # configurations with the same key (two lambdas, partials of one + # function, 1 and "1") would be silently dropped, so refuse those. + first_config = {} + for key, config in zip(vkeys, configs): + seen = first_config.setdefault(key, config) + if not _same_config(seen, config): + raise ValueError( + f"build_vset({name!r}): different functions or parameter " + f"values share the key {key}; give the functions distinct " + "names (e.g. with def instead of lambda or partial) and use " + "parameter values that print differently" + ) merged = dict(zip(vkeys, vfuncs)) vkeys, vfuncs = list(merged.keys()), list(merged.values()) diff --git a/vflow/pipeline.py b/vflow/pipeline.py index 7e79e47..8341e5d 100644 --- a/vflow/pipeline.py +++ b/vflow/pipeline.py @@ -33,7 +33,9 @@ def run(self, *args, **kwargs): A step that returns a tuple or list (e.g. one output dict per input) passes them on as separate positional arguments. `kwargs` go to every - step. + step. As when calling a Vset directly, a step given several inputs + splits its output into one dict per input; for a step that combines + its inputs into one output (e.g. a metric), pass `n_out=1`. """ run_step_cached = self.memory.cache(_run_step) outputs = args @@ -157,7 +159,7 @@ def build_graph_recur(node, G): def _run_step(step, *args, **kwargs): - if step._fitted: - return step.vfuncs, step + # always call the step: returning a fitted step's vfuncs would hand Vfunc + # objects, not outputs, to the next step (the pipeline cache avoids rework) outputs = step(*args, **kwargs) return outputs, step diff --git a/vflow/utils.py b/vflow/utils.py index fdae9aa..aec9877 100644 --- a/vflow/utils.py +++ b/vflow/utils.py @@ -315,40 +315,43 @@ def perturbation_stats( if (isinstance(func, list) and ("mean" in func or "std" in func)) and ( type(df[wrt].iloc[0]) in [list, np.ndarray] ): - # iterate the groups (same order as gb.groups); get_group with a single - # group key changed meaning in pandas 3 + unsupported = [f for f in func if f not in ("count", "mean", "std")] + if unsupported: + raise ValueError( + f"for list or array entries in {wrt!r}, func supports only " + f"'count', 'mean' and 'std'; got {unsupported}" + ) + # iterating the groups gives the same order as gb.agg; get_group with + # a single group key changed meaning in pandas 3 wrt_arrays = [np.stack(d.tolist()) for _, d in gb] - n_cols = wrt_arrays[0].shape[1] df_out = pd.DataFrame(gb.agg("count")) df_out.columns = [f"{prefix}-count"] - if "mean" in func: - if split: - col_means = [arr.mean(axis=0) for arr in wrt_arrays] - wrt_means = pd.DataFrame( - col_means, - columns=[f"{prefix}{i}-mean" for i in range(n_cols)], - index=gb.groups.keys(), - ) - else: - col_means = [{f"{prefix}-mean": arr.mean(axis=0)} for arr in wrt_arrays] - wrt_means = pd.DataFrame(col_means, index=gb.groups.keys()) - wrt_means.index.names = df_out.index.names - df_out = df_out.join(wrt_means) - if "std" in func: + stats = { + "mean": lambda arr: arr.mean(axis=0), + "std": lambda arr: arr.std(axis=0, ddof=1), + } + for stat, compute in stats.items(): + if stat not in func: + continue + per_group = [compute(arr) for arr in wrt_arrays] + # build on df_out's index: group keys may be tuples, which a fresh + # index would turn into a MultiIndex if split: - col_stds = [arr.std(axis=0, ddof=1) for arr in wrt_arrays] - wrt_stds = pd.DataFrame( - col_stds, - columns=[f"{prefix}{i}-std" for i in range(n_cols)], - index=gb.groups.keys(), + # one column per entry position: out0-mean, ... for 1-D entries, + # out0_1-mean, ... for 2-D entries such as predict_proba output + cols = [ + f"{prefix}{'_'.join(map(str, ix))}-{stat}" + for ix in np.ndindex(per_group[0].shape) + ] + block = pd.DataFrame( + [v.reshape(-1) for v in per_group], columns=cols, index=df_out.index ) else: - col_stds = [ - {f"{prefix}-std": arr.std(axis=0, ddof=1)} for arr in wrt_arrays - ] - wrt_stds = pd.DataFrame(col_stds, index=gb.groups.keys()) - wrt_stds.index.names = df_out.index.names - df_out = df_out.join(wrt_stds) + cells = np.empty(len(per_group), dtype=object) + for i, v in enumerate(per_group): + cells[i] = v + block = pd.DataFrame({f"{prefix}-{stat}": cells}, index=df_out.index) + df_out = df_out.join(block) if "count" not in func: df_out = df_out.drop(columns=f"{prefix}-count") else: diff --git a/vflow/vset.py b/vflow/vset.py index f69ed02..28c9d05 100644 --- a/vflow/vset.py +++ b/vflow/vset.py @@ -3,9 +3,13 @@ """ import hashlib +import inspect import os +import re +import types import warnings from copy import deepcopy +from functools import partial import joblib import numpy as np @@ -117,13 +121,14 @@ def __init__( assert len(vfuncs) == len( vfunc_keys ), "vfuncs list and vfunc_keys list do not have the same length" - if len(set(map(repr, vfunc_keys))) < len(vfunc_keys): + # TODO: how best to handle tuple subkeys? + vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] + # compare as the dict below will (1 and 1.0 are one key) + if len(dict.fromkeys(vfunc_keys)) < len(vfunc_keys): raise ValueError( f"vfunc_keys must be unique; duplicates would silently " - f"drop vfuncs: {vfunc_keys}" + f"drop vfuncs: {[k[0].value for k in vfunc_keys]}" ) - # TODO: how best to handle tuple subkeys? - vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] else: vfunc_keys = [ (self.__create_subkey(f"{name}_{i}"),) for i in range(len(vfuncs)) @@ -172,9 +177,7 @@ def _apply_func(self, *args, out_dict: dict = None): # serve stale results, and it cannot hash lambdas at all. Key the # cache on the functions' code and the inputs instead. try: - cache_key = hashlib.sha256( - cloudpickle.dumps((out_dict, self._async, self._lazy, args)) - ).hexdigest() + cache_key = _cache_key(out_dict, self._async, self._lazy, args) except Exception as e: # noqa: BLE001 - fall back to no caching warnings.warn( f"Vset {self.name!r}: could not fingerprint the inputs for " @@ -198,36 +201,40 @@ def _apply_func(self, *args, out_dict: dict = None): if self._mlflow is not None: run_dict = {} - # log subkeys as params and value as metric - for k, v in out_dict.items(): - # only numeric scalars are metrics; fitted models, predictions - # and lazy promises are not logged - if k == PREV_KEY or not _is_numeric_scalar(v): - continue - origins = np.array([subk.origin for subk in k]) - # ignore init origins and the last origin (this Vset) - param_idx = [i for i in range(len(k[:-1])) if origins[i] != "init"] - # get or create mlflow run - run_dict_key = tuple(subk.value for subk in k[:-1]) - if run_dict_key in run_dict: - run_id = run_dict[run_dict_key] - else: - run = self._mlflow.create_run(self._exp_id) - run_id = run.info.run_id - run_dict[run_dict_key] = run_id - # log params - for idx in param_idx: - subkey = k[idx] - param_name = subkey.origin - # check if the origin occurs multiple times - if np.sum(origins == param_name) > 1: - occurence = np.sum(origins[:idx] == param_name) - param_name = param_name + str(occurence) - self._mlflow.log_param(run_id, param_name, subkey.value) - self._mlflow.log_metric(run_id, k[-1].value, float(v)) - # close the runs, or they stay RUNNING forever - for run_id in run_dict.values(): - self._mlflow.set_terminated(run_id) + try: + # log subkeys as params and value as metric + for k, v in out_dict.items(): + # only numeric scalars are metrics; fitted models, predictions + # and lazy promises are not logged + if k == PREV_KEY or not _is_numeric_scalar(v): + continue + origins = np.array([subk.origin for subk in k]) + # ignore init origins and the last origin (this Vset) + param_idx = [i for i in range(len(k[:-1])) if origins[i] != "init"] + # get or create mlflow run + # Subkeys compare by value and origin, so equal values + # from different Vsets get their own runs + run_dict_key = k[:-1] + if run_dict_key in run_dict: + run_id = run_dict[run_dict_key] + else: + run = self._mlflow.create_run(self._exp_id) + run_id = run.info.run_id + run_dict[run_dict_key] = run_id + # log params + for idx in param_idx: + subkey = k[idx] + param_name = subkey.origin + # check if the origin occurs multiple times + if np.sum(origins == param_name) > 1: + occurence = np.sum(origins[:idx] == param_name) + param_name = param_name + str(occurence) + self._mlflow.log_param(run_id, param_name, subkey.value) + self._mlflow.log_metric(run_id, _metric_name(k[-1].value), float(v)) + finally: + # close the runs, even if logging fails, or they stay RUNNING + for run_id in run_dict.values(): + self._mlflow.set_terminated(run_id) return out_dict def fit(self, *args): @@ -354,6 +361,15 @@ def __create_subkey(self, value): return Subkey(value, self.name, self._output_matching) +def _metric_name(value) -> str: + """An MLflow metric name for a subkey value: build_vset keys are tuples + like ('func=score', 'scale=1'), and MLflow allows only letters, digits, + and _ - . / : and spaces.""" + if isinstance(value, tuple): + value = "_".join(map(str, value)) + return re.sub(r"[^\w\-./: ]", "_", str(value)) + + def _is_numeric_scalar(v) -> bool: """Whether `v` can be logged as an MLflow metric.""" return isinstance(v, (int, float, np.integer, np.floating)) and not isinstance( @@ -361,6 +377,55 @@ def _is_numeric_scalar(v) -> bool: ) +def _cache_key(out_dict: dict, is_async: bool, lazy: bool, args: tuple) -> str: + """A fingerprint of the vfuncs in `out_dict` and the data in `args`. + + The data is hashed with joblib, which hashes by value: a cloudpickle of it + would change with object identity (an output loaded back from the cache + is an equal copy that shares objects differently), so downstream Vsets + would miss the cache. The lineage entry (PREV_KEY) holds Vset objects, not + data, and is left out. The vfuncs are pickled with cloudpickle, which + pickles lambdas and functions defined in __main__ by value; functions from + importable modules are pickled by reference, so their source is added. + """ + data = tuple( + {k: v for k, v in a.items() if k != PREV_KEY} if isinstance(a, dict) else a + for a in args + ) + try: + data_hash = joblib.hash(data) + except Exception: # noqa: BLE001 - e.g. lazy promises holding lambdas + data_hash = hashlib.sha256(cloudpickle.dumps(data)).hexdigest() + h = hashlib.sha256(cloudpickle.dumps((out_dict, is_async, lazy))) + h.update(data_hash.encode()) + for source in _function_sources(out_dict.values()): + h.update(source.encode()) + return h.hexdigest() + + +def _function_sources(objs) -> list: + """Source code of the Python functions reachable from `objs` (unwrapping + Vfuncs, bound methods and partials), in a deterministic order.""" + sources, seen, stack = [], set(), list(objs)[::-1] + while stack: + obj = stack.pop() + if id(obj) in seen: + continue + seen.add(id(obj)) + if isinstance(obj, (Vfunc, AsyncVfunc)): + stack.append(obj.vfunc) + elif isinstance(obj, types.MethodType): + stack += [obj.__self__, obj.__func__] + elif isinstance(obj, partial): + stack += [*obj.keywords.values(), *obj.args, obj.func] + elif isinstance(obj, types.FunctionType): + try: + sources.append(inspect.getsource(obj)) + except (OSError, TypeError): # no source file, e.g. in a REPL + pass + return sources + + def _apply_func_cached( cache_key, out_dict: dict, is_async: bool, lazy: bool, args: tuple ): From d6a346de32273c82c6c4a2213444c37f124275ed Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Thu, 15 Oct 2026 17:09:00 -0400 Subject: [PATCH 5/6] New vflow logo: drop-ring wordmark, mark and favicon The docs header uses the wordmark, the favicon is the drop ring, and the README points at the new wordmark. logo_vflow_straight.png is kept since external pages may link to it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Xy8Dg7fAqocHGzMqVCiSK --- README.md | 2 +- docs/agents.html | 4 +- docs/helpers.html | 67 ++++++++++-- docs/html.mako | 4 +- docs/index.html | 4 +- docs/pipeline.html | 24 +++-- docs/style_docs.py | 2 +- docs/subkey.html | 4 +- docs/utils.html | 122 +++++++++++---------- docs/vflow-internals.html | 4 +- docs/vflow_logo.png | Bin 0 -> 35476 bytes docs/vflow_logo.svg | 1 + docs/vflow_logo_dark.png | Bin 0 -> 34926 bytes docs/vflow_logo_dark.svg | 1 + docs/vflow_mark.svg | 1 + docs/vflow_mark_512.png | Bin 0 -> 14683 bytes docs/vfunc.html | 4 +- docs/vset.html | 220 +++++++++++++++++++++++++------------- 18 files changed, 301 insertions(+), 163 deletions(-) create mode 100644 docs/vflow_logo.png create mode 100644 docs/vflow_logo.svg create mode 100644 docs/vflow_logo_dark.png create mode 100644 docs/vflow_logo_dark.svg create mode 100644 docs/vflow_mark.svg create mode 100644 docs/vflow_mark_512.png diff --git a/README.md b/README.md index a8163b8..c0ea07e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

    - vflow logo + vflow logo

    diff --git a/docs/agents.html b/docs/agents.html index 470fe17..d010819 100644 --- a/docs/agents.html +++ b/docs/agents.html @@ -15,7 +15,7 @@ - + @@ -32,7 +32,7 @@

    vflow · Stability analysis for data-science pipelines, with PCS.

    diff --git a/docs/helpers.html b/docs/helpers.html index f69b584..b47adfe 100644 --- a/docs/helpers.html +++ b/docs/helpers.html @@ -14,7 +14,7 @@ - + @@ -30,7 +30,7 @@

    vflow · Stability analysis for data-science pipelines, with PCS.

    @@ -102,13 +102,34 @@ def _func_name(f) -> str: - """A readable name for `f`: a function's __name__, the wrapped function's - for a functools.partial, the class name for a callable object.""" + """A readable name for `f`: a function's __name__, the wrapped function + and its bound arguments for a functools.partial (so partials of one + function get distinct names), the class name for a callable object.""" if isinstance(f, partial): - return _func_name(f.func) + bound = [repr(a) for a in f.args] + bound += [f"{k}={v!r}" for k, v in f.keywords.items()] + return f"{_func_name(f.func)}({', '.join(bound)})" return getattr(f, "__name__", None) or type(f).__name__ +def _same_config(a, b) -> bool: + """Whether two (func, param values) pairs from build_vset are the same.""" + (fa, va), (fb, vb) = a, b + if fa is not fb or len(va) != len(vb): + return False + for x, y in zip(va, vb): + if x is y: + continue + if type(x) is not type(y): + return False + try: + if not bool(x == y): + return False + except (TypeError, ValueError): # e.g. arrays, whose == is elementwise + return False + return True + + def _accepts_random_state(f) -> bool: """Whether `f` (a function, partial, class or callable object) can take a `random_state` keyword argument. Returns True when it cannot be told.""" @@ -212,6 +233,9 @@ vfuncs = [] vkeys = [] + # (func, param values) behind each key, to tell a repeated configuration + # from distinct ones that happen to print the same + configs = [] if seed is not None: if "random_state" in kwargs or any( @@ -256,6 +280,7 @@ vkeys.append((f"rep={i}",) + vkey_tup) else: vkeys.append(vkey_tup) + configs.append((f, tup)) rep_kwargs = dict(kwargs_dict) if seed is not None: rep_kwargs["random_state"] = next(seeds) @@ -272,7 +297,19 @@ vkeys = None elif len(set(vkeys)) < len(vkeys): # overlapping param_dicts repeat a configuration; keep one vfunc per - # key (the last, as a dict would), in first-seen order + # key (the last, as a dict would), in first-seen order. Distinct + # configurations with the same key (two lambdas, partials of one + # function, 1 and "1") would be silently dropped, so refuse those. + first_config = {} + for key, config in zip(vkeys, configs): + seen = first_config.setdefault(key, config) + if not _same_config(seen, config): + raise ValueError( + f"build_vset({name!r}): different functions or parameter " + f"values share the key {key}; give the functions distinct " + "names (e.g. with def instead of lambda or partial) and use " + "parameter values that print differently" + ) merged = dict(zip(vkeys, vfuncs)) vkeys, vfuncs = list(merged.keys()), list(merged.values()) @@ -586,6 +623,9 @@

    Returns

    vfuncs = [] vkeys = [] + # (func, param values) behind each key, to tell a repeated configuration + # from distinct ones that happen to print the same + configs = [] if seed is not None: if "random_state" in kwargs or any( @@ -630,6 +670,7 @@

    Returns

    vkeys.append((f"rep={i}",) + vkey_tup) else: vkeys.append(vkey_tup) + configs.append((f, tup)) rep_kwargs = dict(kwargs_dict) if seed is not None: rep_kwargs["random_state"] = next(seeds) @@ -646,7 +687,19 @@

    Returns

    vkeys = None elif len(set(vkeys)) < len(vkeys): # overlapping param_dicts repeat a configuration; keep one vfunc per - # key (the last, as a dict would), in first-seen order + # key (the last, as a dict would), in first-seen order. Distinct + # configurations with the same key (two lambdas, partials of one + # function, 1 and "1") would be silently dropped, so refuse those. + first_config = {} + for key, config in zip(vkeys, configs): + seen = first_config.setdefault(key, config) + if not _same_config(seen, config): + raise ValueError( + f"build_vset({name!r}): different functions or parameter " + f"values share the key {key}; give the functions distinct " + "names (e.g. with def instead of lambda or partial) and use " + "parameter values that print differently" + ) merged = dict(zip(vkeys, vfuncs)) vkeys, vfuncs = list(merged.keys()), list(merged.values()) diff --git a/docs/html.mako b/docs/html.mako index 97cc2c7..a89c01c 100644 --- a/docs/html.mako +++ b/docs/html.mako @@ -402,7 +402,7 @@ - + % if google_analytics: @@ -30,7 +30,7 @@

    vflow · Stability analysis for data-science pipelines, with PCS.

    diff --git a/docs/pipeline.html b/docs/pipeline.html index 6e87df4..e43d4b5 100644 --- a/docs/pipeline.html +++ b/docs/pipeline.html @@ -14,7 +14,7 @@ - + @@ -30,7 +30,7 @@

    vflow · Stability analysis for data-science pipelines, with PCS.

    @@ -87,7 +87,9 @@ A step that returns a tuple or list (e.g. one output dict per input) passes them on as separate positional arguments. `kwargs` go to every - step. + step. As when calling a Vset directly, a step given several inputs + splits its output into one dict per input; for a step that combines + its inputs into one output (e.g. a metric), pass `n_out=1`. """ run_step_cached = self.memory.cache(_run_step) outputs = args @@ -211,8 +213,8 @@ def _run_step(step, *args, **kwargs): - if step._fitted: - return step.vfuncs, step + # always call the step: returning a fitted step's vfuncs would hand Vfunc + # objects, not outputs, to the next step (the pipeline cache avoids rework) outputs = step(*args, **kwargs) return outputs, step @@ -371,7 +373,9 @@

    Classes

    A step that returns a tuple or list (e.g. one output dict per input) passes them on as separate positional arguments. `kwargs` go to every - step. + step. As when calling a Vset directly, a step given several inputs + splits its output into one dict per input; for a step that combines + its inputs into one output (e.g. a metric), pass `n_out=1`. """ run_step_cached = self.memory.cache(_run_step) outputs = args @@ -447,7 +451,9 @@

    Methods

    (the first on args), and the last step's outputs are returned.

    A step that returns a tuple or list (e.g. one output dict per input) passes them on as separate positional arguments. kwargs go to every -step.

    +step. As when calling a Vset directly, a step given several inputs +splits its output into one dict per input; for a step that combines +its inputs into one output (e.g. a metric), pass n_out=1.

    Expand source code @@ -458,7 +464,9 @@

    Methods

    A step that returns a tuple or list (e.g. one output dict per input) passes them on as separate positional arguments. `kwargs` go to every - step. + step. As when calling a Vset directly, a step given several inputs + splits its output into one dict per input; for a step that combines + its inputs into one output (e.g. a metric), pass `n_out=1`. """ run_step_cached = self.memory.cache(_run_step) outputs = args diff --git a/docs/style_docs.py b/docs/style_docs.py index 2f4ed30..d5e65d8 100644 --- a/docs/style_docs.py +++ b/docs/style_docs.py @@ -25,7 +25,7 @@ def drop(pattern, text, what): # the readme opens with a large centered logo, which the header bar now carries data = re.sub( - r'

    \s*]*>\s*

    ', + r'

    \s*]*>\s*

    ', "", data, count=1, diff --git a/docs/subkey.html b/docs/subkey.html index 3aced68..1192053 100644 --- a/docs/subkey.html +++ b/docs/subkey.html @@ -14,7 +14,7 @@ - + @@ -30,7 +30,7 @@

    vflow · Stability analysis for data-science pipelines, with PCS.

    diff --git a/docs/utils.html b/docs/utils.html index 5e05edb..76d36d7 100644 --- a/docs/utils.html +++ b/docs/utils.html @@ -14,7 +14,7 @@ - + @@ -30,7 +30,7 @@

    vflow · Stability analysis for data-science pipelines, with PCS.

    @@ -369,40 +369,43 @@ if (isinstance(func, list) and ("mean" in func or "std" in func)) and ( type(df[wrt].iloc[0]) in [list, np.ndarray] ): - # iterate the groups (same order as gb.groups); get_group with a single - # group key changed meaning in pandas 3 + unsupported = [f for f in func if f not in ("count", "mean", "std")] + if unsupported: + raise ValueError( + f"for list or array entries in {wrt!r}, func supports only " + f"'count', 'mean' and 'std'; got {unsupported}" + ) + # iterating the groups gives the same order as gb.agg; get_group with + # a single group key changed meaning in pandas 3 wrt_arrays = [np.stack(d.tolist()) for _, d in gb] - n_cols = wrt_arrays[0].shape[1] df_out = pd.DataFrame(gb.agg("count")) df_out.columns = [f"{prefix}-count"] - if "mean" in func: - if split: - col_means = [arr.mean(axis=0) for arr in wrt_arrays] - wrt_means = pd.DataFrame( - col_means, - columns=[f"{prefix}{i}-mean" for i in range(n_cols)], - index=gb.groups.keys(), - ) - else: - col_means = [{f"{prefix}-mean": arr.mean(axis=0)} for arr in wrt_arrays] - wrt_means = pd.DataFrame(col_means, index=gb.groups.keys()) - wrt_means.index.names = df_out.index.names - df_out = df_out.join(wrt_means) - if "std" in func: + stats = { + "mean": lambda arr: arr.mean(axis=0), + "std": lambda arr: arr.std(axis=0, ddof=1), + } + for stat, compute in stats.items(): + if stat not in func: + continue + per_group = [compute(arr) for arr in wrt_arrays] + # build on df_out's index: group keys may be tuples, which a fresh + # index would turn into a MultiIndex if split: - col_stds = [arr.std(axis=0, ddof=1) for arr in wrt_arrays] - wrt_stds = pd.DataFrame( - col_stds, - columns=[f"{prefix}{i}-std" for i in range(n_cols)], - index=gb.groups.keys(), + # one column per entry position: out0-mean, ... for 1-D entries, + # out0_1-mean, ... for 2-D entries such as predict_proba output + cols = [ + f"{prefix}{'_'.join(map(str, ix))}-{stat}" + for ix in np.ndindex(per_group[0].shape) + ] + block = pd.DataFrame( + [v.reshape(-1) for v in per_group], columns=cols, index=df_out.index ) else: - col_stds = [ - {f"{prefix}-std": arr.std(axis=0, ddof=1)} for arr in wrt_arrays - ] - wrt_stds = pd.DataFrame(col_stds, index=gb.groups.keys()) - wrt_stds.index.names = df_out.index.names - df_out = df_out.join(wrt_stds) + cells = np.empty(len(per_group), dtype=object) + for i, v in enumerate(per_group): + cells[i] = v + block = pd.DataFrame({f"{prefix}-{stat}": cells}, index=df_out.index) + df_out = df_out.join(block) if "count" not in func: df_out = df_out.drop(columns=f"{prefix}-count") else: @@ -1072,40 +1075,43 @@

    Returns

    if (isinstance(func, list) and ("mean" in func or "std" in func)) and ( type(df[wrt].iloc[0]) in [list, np.ndarray] ): - # iterate the groups (same order as gb.groups); get_group with a single - # group key changed meaning in pandas 3 + unsupported = [f for f in func if f not in ("count", "mean", "std")] + if unsupported: + raise ValueError( + f"for list or array entries in {wrt!r}, func supports only " + f"'count', 'mean' and 'std'; got {unsupported}" + ) + # iterating the groups gives the same order as gb.agg; get_group with + # a single group key changed meaning in pandas 3 wrt_arrays = [np.stack(d.tolist()) for _, d in gb] - n_cols = wrt_arrays[0].shape[1] df_out = pd.DataFrame(gb.agg("count")) df_out.columns = [f"{prefix}-count"] - if "mean" in func: - if split: - col_means = [arr.mean(axis=0) for arr in wrt_arrays] - wrt_means = pd.DataFrame( - col_means, - columns=[f"{prefix}{i}-mean" for i in range(n_cols)], - index=gb.groups.keys(), - ) - else: - col_means = [{f"{prefix}-mean": arr.mean(axis=0)} for arr in wrt_arrays] - wrt_means = pd.DataFrame(col_means, index=gb.groups.keys()) - wrt_means.index.names = df_out.index.names - df_out = df_out.join(wrt_means) - if "std" in func: + stats = { + "mean": lambda arr: arr.mean(axis=0), + "std": lambda arr: arr.std(axis=0, ddof=1), + } + for stat, compute in stats.items(): + if stat not in func: + continue + per_group = [compute(arr) for arr in wrt_arrays] + # build on df_out's index: group keys may be tuples, which a fresh + # index would turn into a MultiIndex if split: - col_stds = [arr.std(axis=0, ddof=1) for arr in wrt_arrays] - wrt_stds = pd.DataFrame( - col_stds, - columns=[f"{prefix}{i}-std" for i in range(n_cols)], - index=gb.groups.keys(), + # one column per entry position: out0-mean, ... for 1-D entries, + # out0_1-mean, ... for 2-D entries such as predict_proba output + cols = [ + f"{prefix}{'_'.join(map(str, ix))}-{stat}" + for ix in np.ndindex(per_group[0].shape) + ] + block = pd.DataFrame( + [v.reshape(-1) for v in per_group], columns=cols, index=df_out.index ) else: - col_stds = [ - {f"{prefix}-std": arr.std(axis=0, ddof=1)} for arr in wrt_arrays - ] - wrt_stds = pd.DataFrame(col_stds, index=gb.groups.keys()) - wrt_stds.index.names = df_out.index.names - df_out = df_out.join(wrt_stds) + cells = np.empty(len(per_group), dtype=object) + for i, v in enumerate(per_group): + cells[i] = v + block = pd.DataFrame({f"{prefix}-{stat}": cells}, index=df_out.index) + df_out = df_out.join(block) if "count" not in func: df_out = df_out.drop(columns=f"{prefix}-count") else: diff --git a/docs/vflow-internals.html b/docs/vflow-internals.html index e3c8f6e..ae43160 100644 --- a/docs/vflow-internals.html +++ b/docs/vflow-internals.html @@ -15,7 +15,7 @@ - + @@ -32,7 +32,7 @@

    vflow · Stability analysis for data-science pipelines, with PCS.

    diff --git a/docs/vflow_logo.png b/docs/vflow_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..bd9ce33327fca4a20c208b77cb8254d9ceb04871 GIT binary patch literal 35476 zcmd43i$Bx<`#)YfNJ8aQId!B`RLCh*lAO<`FeItul-AhTsHAeLB$2b6Hq0TXF+?b0 z(S|UGB+Sflh8e%d#OwEY{|Dc0w{G3qbC2iae7~;ie%-IVFg4cY+j(H;rcImp^sZeo z-?V9K{-#ZvOLlArpLkwG!@xg0Ue~VsY}&;Ch4XiFv@E~arcE!e>Rq{H=^Ia@@W$J@ z1guUQ9o&(s^LBS{f~u=t2gR?I)=Ik&`Lfve^G<79`P`Sqzu6va0ufK|8&M7}(( zinPSf8-sr9ePPnUDCLuxS$*NO-AP@LzKNoDd}Ce)EQ|6sltI?(rfKMVe=rm9nc zp8DkE@dT+AR_nKUeogw9boy2XO0lkQih5PYx47DHEP$%{Z|o$axhz|0ugsS1Mh#UI zf!_K-Er$uQinBQm$e@Z$)Zj}(ox(gT(K~H0$#b?cNRZ5OFo`|rd#fx`EBYCPMgP)N z&dg%{o3ZJ((&KJB*(gVcy#$$W3iAeLKYs5owMS;YV%g7qwUzBn5T4NPB94(>J(_3@(42*eCc%GF;)s$?9 zIK%xeoiJr`uEeHIpURKVeVhBG722md%eW^BEKM{la0fCcuxkuPMlDszdGNcjuYjd>3_6R$MVdXJZs~bT0zIx!KTklSWuZE4vqj6F|HslE`ktn**rTso5;-GX+%)XAH#cFWQR?ZH|3h%h_?HT{ldnZZ`M zSd-MMwbK%KOSP%_SLcI9D}weu=4M9C)#Z4$EKL61j(oZ`^WQ}{dPVxU#!p_F5w#{& zSazqA8j_+%J=?mSK1n8Ir7!F~wAQJ+T+h%uzlJ=k5FvNdNz+K!oM!gcxfJTm()QIW z6!o?5Bzs#v)Qu1=Ly|V@sK0gX9U*Oz^k{sK7f@nEY~tm7C2&3pAjOFHXX%hLL9Ew^ z-Tdhp}bky_nhBFG^&GP??gbq<^h89gP(gnoTUIwhY-bVHewXGv)#R z!d9ELh6l4N;{y^nNAh6m1eU02q$=RQ@lM6i*~JjQAcCR9MN{NSH`<4{v!7*ZxYpX~ z{bmXqGs0NqYW~TRSpXM4`aIs~O-EXJ!ID_kLLCTFD=WM-X4T|3DCqcaL63etz-*XG zMQ5r#Oy(E-@`p0kRKD?WlM=DZfaJ7A34Z#zwaRr>t&Nc!e5fezW`g*YOV#_m{-RBM zrC${u<*MCEj@Vthm99&T-FOoCH?&rkX#}geXa%0khf;t2_vAK}_E3Pq#gc%PU9{R) zyET0^Gf_cvv&h(Y$tAhNLHfniMUKH7p-g>D7!n7up@@Y{%a`!FyD-OyXCf&mj%^xKItRa5d)=-r@J9=P4r98 z-bqr>W+jvNH)*)#nyRDXdE2Oq=BGE_{srJsu-d6){vF0)1cjVQWRN|_W)j~fO7j0S zqxzZ4{`bw-N_0PJa`Ny={D+n<2V}#_!5mmyXh$8I{d>v*jb0$4WBBwvcdRZdK^?}n zaa;8zB=S7g#f)4b{I{mo2?_L+s)AK4|IT*n#Da?Dnm(Es++~3nv!KfQI~q)dn(h=C znH8{zbZg(eew^c6!fjLddEP)1y_ulXFUs^NVB6YxUoJ*~)5PdX=UG2pS$ZRdD>@>tscij?yo?v3eOfm`qk< z26>}Xj85d|3BSlF&7-fP3z&!?f|?sG^xdfw;yB4D>#OT04t!BiPiDA= zJmRUj2LHsy9kr})uJ}p*9GF3C0XE>m{m*UGvv-s?PS;74V|5Du_P7I=_py?|F7m?z zg7D0Vm+KeY@;fnl#YSSM;@L91f?KbcdfAZvIzVl*{0{7in$R6%-(k(3=w=OliYM*T z`jAfk-j-gY@Te3#zVGFN{xDHQG-V#(KC%ue!r!*fJ5w(vr?~V+Qz%a&O_}>c4zEAc z{o}Vq2F%n}Vu#|R!J zos&wsr1lw&|U8s&<>fKMRN>$rk=MNfsc6}vhmiN^W#r~*aWpr&m|6i z*Hr5&xJWViqP@Nw{DX{WXmZs8PTRLwwg-a90WBjpTGlFiL@)i;LIn zsG%!1F0`lkkSwkLK8`8ESvbr!W&>wBn|_G8F%jB`SgC$Mn}d&yWAD77v`_s1G;*cM zlSSFZ+&1$MV5>j&ZR}EqUO<2VGmMg6HHs@P{5;@8*0|!lG0*ELkEs__-j%o~k83DD zDU^U5e8`<$8%L&@x7U^=x_p)^2TNjW5B=8&>IlKeFSzC2Z>_pa(Ng4JM}_yp+vfYp zqRYkB8Kqt6bg2UyEAxFOoRzTjxH{w9d?&RprC`T8ObbCZZUhuKAOnZ3vsobx8`nc3 zasLGi@VT4I7M4Dkip}~|lIwu?k}W^)mYZi+00A9~tam&*6~@kAx>=nOcrd?OKza4} z|1{$5jUkJ&l%fmzEtve_y6c;wc3~(F;y~*Z+P#&Y1_m{~F{lu!u3Ty;JFe+1G?y-M zV14ZaOGZSp<>z>IoRL~zoo+i27@W{Kzn6jLOT7tKIB8Y0l(t zS%-{^A7+>I|F>&@XBp5yDR_(O`2T!8Mq2;xGZvB&j3T@Ao`?1SxbWWu89FGV&w<__$N}238>`WvrcTi& zi!Lg4xPy)HTi=*F&XKgS#UuY36s%^0F|D7k!1M3=HZ`;DDC&v=>nFlJY!mO@i~qLz zqS%p|%Lo2@`HuWo+c&yD%XRAG=Km*}ypIM3{5J4fiUQI{uE_Kz5ai(cCU?CTR@L+g z?iY`pi~a6xuMupRK@{P@GsoU3aZMlTB4z}k>37?`&mz;OeP#s`;H&IR)Qm`s$U9YeFX4P*R3(4SLWRV zqU{6FGb;u{{8^WK3gxK4^u>goku<_ky(%D zKx|bpD&HW6mlk6xL4&yzA#e9wiQII4-oLp|oX}`Xz&&p{J!3dVfez5`w%7k*x1A26 z60VLve;n-v4Jf)R`k{fXJ2wMopp?bdmw}p>@Bi~#J#RBkewPU<_l(_&wo(we(y8O? zALh$Ccw%0k*2mh~sinc$7Ik=jMP)pTn_;{qSH9_R?~=4+PeoA!!Hm$EWk2VfZ~f40 zQC^}_anIR=H)1po{~PFqde3>bhx^m@dBQBp@|Qn?3C`4d`RNiaV^!%*_?L~mXG^EJ zl)W4};~mRf*yX`n(q2cMiG6jW|^P> zV&-gC+peuP*R}vpKkHi!cS(ZHzF6^-yJBQM9I%skDAVyLuXZY*^_ls1=v=_dcoZ${0FN`_RX#s*sS- zz6q*d1$t%se1~8RUXT=UR}tV`5QxPrUFIXOqvJtGqIxU=KW2pr!OOo=?^b?vnQ&gp zOU}DGNmj#wB9turw9`8#D{|~fq$lzPUgo`ShtN{MP)W8-(-Y|y4=v?|D=^Pj{Dc0B z`G{7@8}k*pE3nHOUEFT@ZWcWFO*t4AqM48QkzjKqsv@*5O5VJ5hnk+B`JYQ82=#vs z==N(0TCV*$TJNO%a~&iJ3k`hf+cXa%vA% zT50F{!Tu|>?Q;F#O!15tW=6dL^;#-U^(sYKF~+l(MRuF19EY{4!6E|{cM`w#V>D4C z*gQ@6GrX?vZemwbmP|vX){dZxk+;2$rFfHT8Ba!Jk4*@oa?=QD; zT(zeh6F(>@B!x>95;l&>iFq1@ZbB*@c<}4Y2i|SzEI3Pi^X=CM}tQ;Uj=r z35UMTgGS2coP|{f3aOsFAhkNVRlHQ#PPxlLrw8j{(=*;QEGbM#($zS0GEpUnm4$b` z_I6uaf)Um4di^p%pOSy|Ah0hW^1tKbG5kHX_G?D9eAwGbD{d1Qc5ntq4$v7|Y*@)f zoX+!Y;qMLjC0S}nQ8>s~J6`5PYP_!`9H3}ODJmJ^o=>2@Ftz`c%OTewOulcsDQzE< zqO9Uc``~C!NQ?JmT>XavKN@z{eMsC*Sg?~+QTi>H!>gQ-L6<)uTlv*K8ybqu&hZ9| zv;E^G-^%-@QaG!GsBQD&Fk*M2D@}kTm_mIK1zdPe2d(|Z)WxsM$7(LCafYu=(N<^_ zx!NQpx66cX&TYiUoBynN(u%o1d{|691-$q|br}>4eQX}8MYEK`ktKTOlSlz>NR*?e zwaL>1(yigts!KS9$x1IdnmMr%)7h7A~>&7-`zA4IUUxMlNeImV$b}g1{=%@+wUJ1Q$&nd z*{*#fW9}Tx!7Zof4OJcfrBxuBi!zU%;I)muuEem$r7g^#I-& zx4&u^IW=`woS|m-v)f%9zMad*c68gL8v*eh5AP!Z`hMsvF{X|q%Me^7=p$VYKZ6hX zF$J;#OR6?Qsjf8qJ*F}srLJ2nxq-CBl-6qW)W=cL6HD=mz|y?S9t}QL)n4eA9}z%3wz{Pa`VLe}$~+scel;DhjRe?rbWJ)wkD~ z?4RO-TM41g&)66W0wZzyq}Iccx}8Uc#mI-=6fT8iip>Z2FK0F;y_S+#HC;^?BhpvD zSC{t2JdKi5>xY27!1b%L?!#6CkG_OH%hS|!%QeiK69WBPmu0CjwmxU1?JnzoTJibx zkb8r`%VUh*P$kcBHu1=WGssBzH3S$CaxItKJYN+z@akVuh#(%F<;nk1yR_mxTbIU! z>O}s`t~;v3$L}6+B@Ll|29Y3Uqja{Y`l`$d4ozNw%wik*MORi6m8QN)Jzzq%xcH=t zVmk-B(N8E@c=W}+iw%@i&X_>TXj$i(zKVAag6m6eSOGelek870dET5+8EQ57SfjH_ zC*{Fh>(<5d=BFXZ>MqreXxaJmv%W?gs?I$bUH`{|w{CdVoLj#ohvVZ!q{KcW@Sdsu zo8i3us!WNL&`k+bHdgvk2yC`ybRly#P0S%`b#7tER|dic-tSh1n)pyD^AT94kJAY5 zcbLeQ?q>Sv0Hw%W*AGw(sKZ|toOaOQB%>>{58(ouhY^{9h`#!`BA?1H6yT z=ALWC9)Q)7|4@XxLx25Y7w4c}8GP~~LO#q8D!xsf>XogxxnorkpSdM%Nb9|9joSSz zB{dY>rvH0nL{d!k9R6q8?wq!sF3h`Avhg|45a(!bYl`+ky;`5P(yx)tH;CU(`o7uD zCjvD@TWIXLX`D(~!`~xuN7nGyr=a;Qls&-KpJhp8%~_Sv`}-pfz2n?f3*c@({e5jQ zlLG^rtrw|RnT2Tbmk=yZK0m?0ZCQYwHY8SyUI|&ogrnQNCyI5I5g%+{h;~(w-Mn+7 zmFQfY^TQJv%10e4oaY!!+(dD0aFJ+PB!=}o!)AA4O)nx7XZrGlqh{ zr8npclW(*66|Zif)%iQxm6pIgVWF?ZXHHD&1+{{K#Z*6$im-|Y{VThnW_nqVb{25pp&~YR z8*mrIcGeA7kWwtz7=O2-Zsjv-DBUC4Ia_LTY}4e$AGmgGsqN2ZW_A9y8TU0s5bz-v z_xc90Q*nwV~-T91Ip7){+f->uC{UsuTh_GMrt7?p>a`&5*6W_j~y^ zw~fHo>qFK1_a*q6Al?<`r*^?-vM-R(%paanNfneWMWd+NAk{bQffE)5W8$OyBb-OM*Wh}gY z_@{auI#JbJ_?-zGB6N=--*0Z}i9WUjG1M$pv0F)3OIEw=E}PTUD>Ft+#BFQZlEc2# zfq5aeB+o8wJc(#~*e06GlK+`noV)ZgLm&3O|D>;37myv_D zjV8?VSPD7bspAJpAauk;OPipjtEf9Cv!D3@R#cBPBEt4KrtsuDQX}dJSiUUtzi(oPb~b0q%k46n zcrIusVcwj1UdX!lSoxEzlWw;oPJ+W{vro=M`e0eyi7-RD9Pquv3y*Dy@$i44nIAV> zQ0qCLxBN}7X+<&4RbSo$_s#p{1<}fj6zz+L;#b8;jSe|J1${y3oaN&%KPbmijLbC} zk4V>K7V^aczJgn7Pe8GbM$*lg0>~BC&^G#(8L-7WIcZH&BYJI&kFQTxGzx`ecw*eV zU|!FQ31$Axs#JE)6`?=EAP5YtumCbY_Uxns-BP{$O%Vc zJAQM*4`P=+6;}S2(Obp=jJVSXxTVS}91~d$VA+KlhFA_z>0GQr_ag7MXp{25w>Z{N=mNDX?`P5cN<^WlzO3@<)ml* zeD7;UA+Y8R8GU|A7yNX+7g~RkE2>TAxL|&!O8p5o*p-RA<@C;k6uL#)(`vKM?9So1 z&e^F0$9h*lptwUZsuO|;TIveDE{a7y_q)Z_qfk*smC>)1dR`M?Niq0R2x$0Rs-I8= z|23M$$}k@$p{Z+>K?+cj8>S#kI21(ul5>L7WxyBnmpR}(n)QQRrR_an& zDD3-R`X!js1zRQB7HLMhtGFGLVQ3D#0@ zB`Mrsi7qSI<8Z`5kT!`x6p~zOhiPtP!WH-0`rr2$%h`30;mj8X6rQ$~g~)=d1TaJ+ zoyq-XxGjvof{sp-Y}u02viwHw)=weGEjbh?b}U}N`+Lx5vpA$MwV`ClbU*}9GB z@|d6NJ#Fr(Tk)dRiPUKK{QyX=L|=dZSLzuz>L4&mS5pxQek19N+Y(qK4vr9dPv&Uz zd+J0=*1eNavKl(&{ECl(Tcj$asp8}Bv^dv-nYxF5uA~Ulo%~^+J1bnSq@Wus4A;y6 zc=D^Qa~JnBnbka%bAD|r2NLbf#O10}@>7>x^^k^B*Oqh}fwi*`4zWoSOoA}6eT6~n z_Xzih;=B=B5n1ED8LytCeyeWCMOE z)0GD85azBpw*;87(`{CKpmW||XQg~saBz2~vlnOBQ4xZxq`T2LhRXww)ieQ=V}Ev^ z0Z zYj#%Vi2-uRDfIdPFr5OdrP+NW8V=1{>l?A~kK~Wb5;f{+^2aftV;#MRzHNjfn+0Y5 z3Ops)Q4A~(C22`7PyfnER4y2?A3*sIcmy}L8<2Xgm{~=8qw{WTKLDV*)=8+ff95{y z4kLM!8)Wt6i8MOz-%j*cL(y=GrIM1`v>=Q~J*nofi$Hb7ILNzS1TxXQBmjF9_o9rTJHV!F}VdK03o;-eeprj$=}1xZox#?xN4% z1#y67>R^3JcJxd>)eG))ItdRS)3LolukXptKtEg#p_*dis8R`Km%(lUy48isP&IZ5 zueeC5Q{;#{_*kVWcZc4}2AC3vcgf$q?|of_OxBpg1kASdmx+<0Waj_y_IlRZtLjq> zz7sj&oL#nVu*^|e{!%m$$XI!b=~$U3dUq+BJ|-j`J#?ZAMFd#qO0pmYfV zLRNY@k&*xZXm%N)uS=M>h&l2ZO^?x^j!RMQkWwtkN(f337((V$@3b1@fW%PgBX}7b zq?68^HnyW0n?}G@ph@&QpDTWf)sM>(E(V0v=Y)c!X85|Lly=-(Z~Ks=i;+KDN~eF; z^=)v65Tm+WXz`^Zx=Az9o>DqirimvA6|s%zB4EzD+y)-I8%vltK$-Zuv96^0u;E16bYLl2o`B=a@}GSf@>@2fDd(G zk?$WOrGyD)8rA|~zi6U#o`S!nKX_@M5IOfGhrD{!_YFO4-df^1f4+08VU}-H8^`{M zThASPZA+d#@BNRw*%9uSh5cB3h znPYPx-r>E2(;VJ#74w2Jn2DgLUN3 zLqj5ErCF>ux7z$d5n(q)BBx5ZfI4pPnb@1}HljE0{QIp5=}BU+toVT0*Yo7&a`Y=t z8J!l3L2>tAqPncDHlyzYm}FhjXUlXY-_Ifqn%mmvlxn?fJuLw4ri$Ljc8;lV5I~nw z_pu3}nas5GP9W22;~gfm^)RE&;ncG)m8ya?Y$@8LOP|UxG8XpIw_2;L!9tv}JDO)r z3{^Yz5Uwj9bD_QJs>7dav!{zK-5|bi7Lp`RFV|ZqZNg3nb1a7U9RybGJ?7IAF-g;c zq9A^K6*C=!d6IpSE2jn6)emc)`Oj$-5>Ol=dc$HY4aUNEE1v+Ec##yOOh-c#cweLlc*txs1b!Np+xGOrE?sMJ?jq8KHRJ%Xi+pjp|3jxGS zH`IG^4ogZAf;$#Vmd?n8raE4eFfJaKu#s3?KHKJS6w^tK>-2T@gp{6n;jgf8iqZ95 z-2;e7)swVzD4GAbjYjF)?xxJ$u($K_0W=4%vVGUWyrx$J5t^j5bE=SOq<(56plYMl z$vuW<+}42Ev(ODiOLoiD+(QNrTlAX&V6a?yoT~#Pn#KYPBTE{z4Bp&fM1?9n_Uw(O z7OQorB^4P{>YiR7*w5r;l!<%yafX(BhmTTyyCOKW9 zEr~eYi>H^w6o$@YN3>~+LS<6c72|_wx&(D1BWsIne53sRFHVU^H8)$^j!PxBiiBiL zT5v|vujTPqLiR-4Y^<1wK$!7;2)Cs!f4+1_HgLhfl{O?;H~yn$-ydybsgRw`+a`Sm z=tHthZeCX5B@MZPhEH(EY@nM^Aa==T$yhHJsvBz-qhH`-CR%xEs!Qd&t3fV4i<*$c zH+f^QZil6;rSBtUA1|nUcCz~or*f9Ge^(Lmr);fHlMX=UDfgXXbDKSZ!(Yo zkq@&Z!hQJQzWZ;9Pu>eBdc9Rw>rK5euNC!8Ty$lm8pxtC1lc8n7hY;|IkSTa*Es7U zyUU1uMMB@osyVajBYi*SCw#0L#vc~33SGn0-0p?@Ajfd5a-q_g0Bu}2-6sJ&39^Ra zyYVtdM&xwF|{OMXOPzRDrX0&ZD_#L#Gkg#dabV#F%VLAZ^a2a+@ z$u-=VW{01V(Otgz;6^(8i0ZH8x$Q84sDl@jLGtJnCXXX;syY^w)IOWT9dU_u6eiGi zu<8WihQ?>=IXw`0fsJVb@*TwTko8KG4S=ba&;L17ki}TwEhVJ2w1)QR!{N+=wHWLM?ZrJ>DVcCzmm_0) zs@2{@9+IjEW8Whn+JB8n9w6+3R+(7ZT9(ZRq^I#sUSxnvZ0&vLUINJK(COH0A{P0q zn*sR0G=40nb{Qbd_Qot2nmBnphQ2>STHW(a7kwd5^^^v(wkx zAX;^>$;o>Gp0?dU8#r8Vbz~p<37KR>T5xqk(gmN?Xx2>O#xIQ`MJlQ@eL*6Oeny#KUaPQlj+UfAP zQcjw_IGqy_T7{TfbyQ=J&lYB4*sK$Pri`Kv+;i9#<8eRK@e2?eU^;2O)tw%e6zy1W zaIeT;fka-KaH+<6q?pfb(_-%<*{%Ds(mUjz=jqiv4Xx<9@6HUx-){QbdwuCFPJ5X4 zL5zYu{&nXn{OXRvtQ&5gUJSX8G7ZkltnzWV4Gga4EjQ)>-#jrLOh z#s%_-qCbE8)rmp=lBK6GXpt<{w@a11^ME7u@5X<^u zIOx|NO@*m<#BtzQJ9t!Ql-k#rJL+#w(G8wjp3MdSJ5XvjW(N+@1>R+$>QmD}g<2NM zg%S_t#)+*f6HomshoVKhOtO)F9Dkc5Gr2z0w{PW+-A8#Ak?Ng&_mu0TsX0^bDxe$8 zOryU=C$|3@H#R3Wf8^Mz)`1@#x=OQCkLyuSDT^;A}CA`{z zL5oFDA}j{WDch)$=Rb__#qZ;+9jnWwmyZfP`IJ?4sgJc!m~si2%eb%2pw2xn%UW^35>$C~1U;orau}1hZjH^#f$a~rC0&)8NlF_^SVEyCPAhM+pZ+9d4 z@-6z9H`b_)D^alF2az*dDzr5hf1C7?snC-1rN@`J;{4;Ct>=8-;iL4f{L2+~8-uVi zggi*v1u@D#rE1+6!i}YUpEr<=#-RSa7 zKU$bDd(ZLCl0Pi+huxypkmrGY30=Qx)NvjXW%pAjCWVL`*}fY*zG-{9au%}d6h}Z5 zH~C)uk8)4hsx|XdEI>_L4X>PEGTES4P7r2Ng@0Qxl8yX@sqOQ48|nFdC;C-WTf!|W zRC*65$7nL9C$ibvy|*b=CpaZrE9(hH_1<{RqHPkEzDIE;NfK=bRy=b2fE|E+aA&4p zB+5e@GY7QSd{@d@q6u(ayH0VUSvs76ZSIoEVs#(oCDa&WS@yHuA|A+30#xbCM-g`u z5PLgPJb2;5$6@leMi3N$7H6@F!iHQ^=DF;D31@5D{Wev@-*|Rl_`y2dp*B9zwyhwu zgP_0^*MsE3>Jg}~5(>RA5*HLLsLEJ58lLo9z$kb`W<5ohzkj)A#ga0KfSk{lvS1eP zPn{Kn;ha0(<3o1O7>$v6-anld6(35>q`P3J!)?ftrqivIoM z9UNPSdE!P*oFc-nz@KF!pWNcP+L&}6aQ?q)Fw?ow{oYC|%~M45@i1s(`8RQYXRSPF z^^oQHefJkD6w^c#v8>+rh46=;J{#>Btb*oD!or-xAa$QgM?HCJpTMw)!wsj2U7lFv zFMUq@^L#}_Pb06OxeJFu#scXtM@)jv6#|j2j?|5b+~J>)UAx?e6eudGL2cu`HR)mw zXV0j4Ujoj%NiaGu3y}q5fC333#yKB7`YB%E(o# z5~8LyrT=PZ)j%fQC#-wFeRURc$5~L*u=mVoo{f+J`3i# zZyS4a;#9xgJ8p>f&;C>V?yB|_-sIE6vyHwSqWD=SD&i*aS@YXc*Ct;x&8oy#hyPha zPL(y}qZhhhuV+^oSP_Ag}+0c8aGvC-3I#&6ya<89lkRB)0MXt>t3I7TB=#V zeVW7)a1@Xqf#yN$h5((-OVYS6)DM_WR?pOQb2K!E?vwVEReFiiuksJp_8wlN*)Dgu zn2&I1HbZs!d8O8yaOuerC^{QaG$O} zbI{0m?8NDLAnJuSI>%?);pk3J#3krGw_)fHXHethaF3^i5w<#+om(DOp+nI=+RUX@ zpOhPrOe2!Q4U0hf^_ODO8Jetoev&!!9N+q$^&CHyv~(#A@N_>&Qnh%s<$mAm#F@L$ zO}t0#d2nihfLOs28K3a{ek)zzdOVfbTBTa72v!k0)QJo(|%S5G5s4FDF(YVwq|d8!rZ~??!jFUb3ztAn8D{?fh|Xk>t5k`)Aq~K<_K~ zd^2pW`nmOv4$z%L2oYc#O%YGue(T0=pb`T3=C#MsDl^~zL%5WUyPW8=j0Ui%4hm;Jg)?t$I`K{^ z7{ySNoV80&m((8rS5AY3Iea*e3Sy<;M_?sq#H;TDU4oQ=y2Qj2{EKVJQ@>YvsXPa& zX%TA*O1ew;Q{Fn&rGik0>&^}z=`L&Ca-0cMuw7(wb@M$c8tk5?7Awhqr2@*(+>|A+ zRgY8VAzE>w=F;!#^QodAu*mM->7okt**q6b`ksWSEaZWr20*M_cHx5G;F7+xd=3-{ z$p4h4{kpnXR-ZF2V2Jk{e^@!moDq9PCc|Hv}^(*5M}mL z#OX*2PT?C>vcuq^ZY(~@XjMWv9gbx|na_Fu(&fngy+yIB=y(h57dN)Gwr+cH* zz~@5; z=aIm_GaO_ozVLUEgDi~v5haBaD+v3Eq0Vz7AKFBhN$fB)^)xpTa9$jN&#$&wQ0Y-| zdp|<)Mp;&_c_gG4P-zZpTJrxoeJ4tAB&aCDT$~n3NZt_>_FdGSzfM$luJ)#!ie;Su zNI;4SK?3%gQ)xHv!9YM+i!TZGlv>1d=$$-zIwlofkj;{c;UuUijcY{XQM$f@l@?3lsq7o!n9ITk<+}AZun=C)BQppC7i=BUd#*?*Z?_;kg z_a!2xICzt|fj14pP>XOZ>p1JQ^1Kw89Vy!zQku=$H@@#5)|}XF{7xWS#!W&bo?>5q zSRSVF_>=rS76wZMnGKijf2(_HQ`R-qL>FXs+?serIEVe)megY*04|jsr=KD~GOWT?Gw zZDB!S0_13pUbQjB8cAgQWUZV7iAd(|6MI)|?2aUTD)=wEp%>j!UNq-y8PfDi8Hw#z}67iIG!};6RU@O?S!#66BBc6nw2@ zj%B1a_#eTPQvDuQ0H(A@-Zd#9P*1En3>l7((G!-t2l>`0b6ZEA`KMW^m?J|5l-{N@ zILC=D7Xg0Ue2C3>12ns@s;nI)YC5;~Axyxnjcj`ogN%8)6Q`5`0`0>LUsA2fm?~+- zS*9LdeaZQfinoV_N$-LSI>Y1EUKntXGZx>`0)$tx{Pii#pVn6HgvKJMSNO5(HoNQR z$WGjCw~4NfJ{Z=-+|yOOEoJ~r=lkqC%RU}asLXN{9lTi%9h$LNj-j@7R;A%8u}GWrZEd{Y$piy^zXp_^#zFGSR&7#1=yhkR)bYP@DEj*zoCqV~5C}`^m>qjRqonf`h~k#KA`$L5 zc5i)Oo`=(LZ>nOodbYPqb&6!HK1VyxV865y13=saluIBPAzr%fl9KT=PoD7X3+;+l z@(2civF?>`8aQ~HLS_D8$9;ayt!w&W<)`BY~P#bK%DovnUoY41yf_&*RV6yAWQE_oaEM^?F@QvQ(j z$w?2D^vq8x7h?L;q5aPB)xa4Ue7(j)^p|O6A)`q%T+AbkeG=dLW?2bTDA^8Z!03l= zTv18UJeu^wTU5ds4j+C=ph7WES*zz6`$%bP^=Hr$u8q@Q^cWUXebB?VIJ1HNg8s*R z(&Th)XwNul962!}{|MfmbnqI>-lHY!W50w@z#n}|+8R*Y+0*vFN4q$rKD7yr0%N%>r>4R5KL|;1BJ%2|EDA9PQ9{{Dw3;WFA5hW`0Jn*)#e*z+_vnUe5av(I_xxQM-EyJC; z)4war0_YX_4XD0?+eXx?9};;5fMQzMH*%&&0q6;+l9I-Aj~~F$<`xat8NLz`G4RZ0 zq#%ytbBtdf$Xw7J`o|5#a(?;X=W=^<0I2P<0Nw2UuN(4E{Y38EWjM##?OafaCgDk464)FOd%Kr zLOf7ex>il`qhV&evGtCnZ*ge}UyhWz86kQiC|$$@;*KqUI`dYZA|Ti|L@7an^<_e9CrVO540Tk$i+Bj;&~6P% zGA$To5?=xNj-l&)?@k71*rsd>rMWrk!tGxYoFAUBW)`;9vi)CFbxzAJ9R(GLxu7DF z6E$qUWAPbZ5TkgZvfF~m$dj_g^!49cXaGXuks-?x_!MV9vXrVU`~LJV)<82POSi1m zKFSnCys$d~i&QwGb1q2TNaNeMc>^aJO$%m~ne~mlAKnj%pdYnh{+}7$X^BgF48x+y zA2-P&Qjn=~4UtEIK+zO8>kWRF0FW+p!Qi`8)W<^b+XK2l*G3vDH7PR^`=jx~EVgDo z3;A3`RPvK@mJC9ITH*yUXELJoVZ+?qf!>0b-64Lu7Cz_nRtS8nJ77w>&-@CRccHsi=@(D*YK7K`&y;w$}H zJma>5TIdoLJIzv#Ds)yGu4zah1b(4WAkL*q3N_tgHv=va#>qS65jYDRqd2L#i^8K? zZNJZh_z^FaecBmkD&Tk&L4H{&I?V}O+P33C2{U6$h0id3;#(I9_i z2BM^1*^L(M=&b%^NLY=23Eh?o9S1Z3s2ZVYa~NN`Hc)dT)7<13DoZCcAl;C)a$d<) zqo;y&Z7PpcX=?T6u_j?8nUGWJRo$0Cch^{V8nNpx$%!L1-D$o~i#U#ASe7sMZ_eP% z#9-1YZX`|Ng^-l`tl>$^^?Hf^8 zL#4vMjJXR(5qrw<0%P(l`(OK>aU`GpPi&MjXj0v;0Lb#ZU+@-9TvzWGLOCINRVF8c zSOhK`rw9&Qwshl%omSqr9@tXqu-w#@TCYU{mA&=AAsh91m`k9N*v(v9Ha|@2?9|Y|tP%X!0w*(z(U38Kt*xLI$yFng8g1 zj9N_-PK-4KXi?RMLjX!oxwq7*Y z(UB+F5duLr0AbRS_yxB<%)9aYzg!|n8JUQ9v?Q&g;ru*b3Uv0B7 z%ROFgN#Q;i(Tgw8vJSBp*qKkKSIrkXlLNA^oCn3Z(AhOcCXP}xg`YrD*&gNR9+iy^ z?JMJ(VMlQAPO-wZ29;Ql1d-Z0-AuPB%<(G3#}w}@T|KrW*#PS0fgI--|N2c&Ess_R z1)|<0y*jK$aW2|d4NK=X#R8v_o16lEYG@C1WrSK~$rA#j>f?6SvxX zaUZQ!B33Eiy?c?8)fR+)r5Nv0#Uh8ALmXH~9VDd`1{qDngmWr!*`c6Sm8@7-q>`ujPe0Tr+`000i#W2Zqw(Nx)=9=^)1-P=z&=2&WtVp0@SbbFY|FA( z)}uAw*#hyngMV{J&9y}s^#}FLp$=Daz-Ej~M=lD=Rg2M{thrH&fd7Bxz2{%k%kw^b zuz*ODCQ20$L1_mNkP=XOQ96QBqJStMJ+w%`LPwH=NS78wszjRfAT=O0NJp9=2|=19 z^zz$y&gc8O|A*U$K6uY&cjlUFu9;m{WBMTmQ&|RD0>`G9eVKmu5yf0%d~CCj9HzQm-_;Cd%${E5_nY~# zJr$xo)$RY^)jzi48)ip^%Ab_4f4O%25RB>=fme?TbU-1lP5R`ziaXP^bK5Tz;8r8R zt)mMBg+I8(v=(31Nx0I~^`ByQ}0E`rP%@a>tP|7z4fMW`()Coo`0v zv5i|e)uxN?PNC=-5sKPp&G6CdO>00ab6>F5fv%gcJ*JAoudbJ?-yBwT#OQx%X07-i z5e~ZSa$GhI{Uf1HGW|>O4Bk^mDZ#JG#GN_ToD#@3n;NzEe@ZUB3Z^^>eAbTYX4?f0 zV7|fr!P;HSaaGpL*d)(*X5~yy(g)WCJ{@~70Yy1#FJX!!F`3By5d&Tchg{s|_LD|Y zUtsvH)?Pw%dUYAQ`=2)pfpJ%LGw?G<))x@FIjN3AkQ3MN)&9MOq<=_fXhuNETBH-W z8j=S;M46{%@9fZ4EfUucTq@wl6M5XbW=Bs;VWsfn9~PB|z~ng`6_hOnE^Ty!=0WE! zwRsl7uyDAi)?Vx-a71;c|9A^q;ooM{LM%m#g+x9(5P)7ylr6|hj-c84r#U0~#o)5vO}{qW8C`{q`j-}+(8G3h(@2ZWk%_XI;%63d z=5T}9ZwFG->%4?L9kR&BraYHYpsi{);=I>t_l+o6YH?<^mid<-dtM*5Z2K$A+nvOL z{@^<8ITmsTJ<+)2?DpI~$>_5<)q}DI6Fg2xxHH)Y>Kbkfd=x%SGh*b2D&n-i49!1c zDmdiDt*!$2BaNeo;Xm-P?Zf*c9-x5|N#Vumjf`;>Zi_qH*;0*hB6P0P&>4Vs*Poj( zt3O|bPg2+ro& zk7?Tq;WM@DjV88@#w*sQ7sgWq>>;>8Anf3=Ul(Yz?)QiM<(CWu1JcPn{y#Ui8iW6y z8Kd`u{z!Ukt5lb88=Rf?zolCF$Zr~Ob6EjQFnB(%a`;wAJwKwI`seF3g2V3;KPT7d zqOHseLQgtm5{4+JXYrv~@T`d{v=wQ{Ta1f3TDxZ6+jyoLfROUm$tSOg_dF;x=KoVv zzvqjq-$;3$*;cC4SiqL}pJo+paY5=hS^0e$W>0t38okPD5}%qA-aIipIecqal?Ifb zl117z%rhU3_@cZsDAAgBt&p zazR?fNa@jIzfjjXDqmW=`9}u&s4D=bWb_-QSL{>OYzlDJ5yy65GFrq)8&y7%%W|{T z4-AAYpZ1U4%Z|30)AkiQq9svZ!~?bq_F0>ufNW*6-LxM$XJsF9KDcXiP-Qxu8h`?%oQ`0D>bLxku|A#)Lx zCv%dH{u(|r60(s>$|kL&SW2hyh@XillkNN~|AEo*`Js?fM^Dt`i6)Gft9Y1tA)E{4+iaeL$4Tmv=M)YMrXt4>j8q5) z4;L*_9eQv!Jm4T&wIFSiwWTYd5d8F?6K*Kxk&obspAu%KPaUOqtFJK6)QAr^o%X&4 zF8T^QKF&p%6mAQ8_)(H#uw|je)5sdCB8%hh2Eq|Vt!?7iYk&Ti`lUJWiq76_G}Pa% z(Z0yqhJ~(wXgixZvBL8%dD7Vocb^5FSe70LiWU+d-p9M$dK%P^eT`lSWJ+FJw@UY)Ha#i!KUW?6bElXcjzw>(w(IQ$qG}g?I0wE2@tgWhj z2n(rz3)dpaC(eKBq?KZ@jvkf^Jy3v^k@94)Rfy@fTg30@EQ@^uXT_8(;EZ;a6CVSs zsEwE_3cnYp{T(6bagk_fQFOzw;5 zvrbzsd*2RIXyqi0t|y536sVl*a?42XCx!pI7kr!>zS>%SZZ|jCfhP9pZ29vK=q6E{p1-A3xccWfmgQlbGSzMc@O;sk8>*T7)&wSPj zPLTx-W>8!&_785r=W2-x+N%lqeMOHXDl{5-rnaKWJ|%^)i@eNyEv5fOp99h`+s~b$ zGHT8+dx#X|9iLZJ4yX)}co5HId;Sw!@w4WWXYD>s7zQ(DaG~!%qNQU=(#)&hXfJ*t zLbPpb=WNa+QZ5-dJ@R_UG?n>Kfmyd2GRS?c)y9AApYfLVc#=1)UWlS!;O3?rA=$_b zmy)%g8@j)D=j|Ey%z*deL*Iz(S0lva6@2Gt;DgUD(HtT<>R4txpH4ry_Gbx((%;i^ zo)62)$mBjd+VNyFshpsYhd=NgC3o<_?nj~5t&-0!E{A^kG2CB2Dy4ENBeKRxJAl8D zU#ns9o+TpdmHEI5lKYwX_u3?T7FG2m+29{fX4EUc&VBE12#NA|wt`T=7=8X_wIXx- ztr%R$K1MAS9SQGfu$n1&mp4~7o{?Z3=%4GO^UaZZDEHyb6C9^Y&Tn#zd=STn7ZVt) zH*ub+*mkWh0!w1wNzPCFU3Q^&n5T|wmXvlw$Z6;1(gX#lVFzQyTI6ReSfiyVr zYR{2RRZ30-8c>>wE_Rn#U^J_h9Io%=u%MMYJJ%OZy}KRnxE*(TxtG(GacZU@hii7@ zzM}N|y$Am}#uEs1Y$2Dys-NqV)N$)+V(__kZN_A^Y2uH>oz>q~x%+l3rDwQuM#IG7 z6T_QIYAUem-NEvFQ;T}b1Zp zb~>I7dU} z8E>9ehouX*jg0%Yw2H5Yo1!1(!9!0N6?MkU&wFoZD>q!}Qz60)Y7nZLE6dVanpAEV&j z55)#aQCBpG_4&zqw0;xA%N?lD(K6iXQ$X%YiVnI{)?6;$l1TIuG;nh2*Z)S@D6n1q zd#XR5YW>@4>cUyd@~O&-#er@AYD=FU=f+tFm`!e&OQ^`<sX)|DIL5;8G#(!b_h-ocGoo}3&|EX>fmGu%VZM>NZr&Yi+9(m1S_kca!YH8O>dI~89 zXI5m4p5#%|D%hG*SHU1M89W3uT@+fk;&%Ka>V*5=SN8Qjc0cB94+BZ$M-M}L>}iSe z7j8@1wsPk`z0N#6pXK>|+A94k@rW%RB5OYuZqFlRe_S$QHPh2EOWe}_H7Q+m(Qxy6 zS}!JSH70agcYDkC{luH+Tf+)Tbt`o&`&M$9T6ggY{UStm$)xp{>apUYrTOQ-?6Y^D z8kxr>vd0Mgcl^aWoBTH_pB;VZ^^>-j9II@FuHU4Yf(n(#W@fGFVpv#9Lo-#k##WPm zJ#<#+)~S~CIV0ahD&rtI2iE7t5o%UD46!HDIGe8$KWo2sX}YZMRHe|bVrG2PSajCO zu(OMs|AB(c?VF)OAYE?HchL);~j@+vEDw5%hwZ>$PjnG^%Mumq;A^%tZJ& zd0niti=WpLDdIE4nG1vgG`=Ae_{|V$dQnQNTHL^?arWK)eq$jwBF|tRF+mbu@VJM| z2))=sY4P1aMDrsV`;qfw*?SNF#o$L-LF{t%1k{XOlNB8gu7ed*wR;n(>V?U4OP*x5)AtFW<&Y5N=Kq>anv209U< zqO)PKy!S31IS$#@XvF30v&#FcvGMSq!6&}$}MRv}eUJEW&zb9RD zVLJ27!wCV(!R2zDyG7f%#&y<4-)+~o?)zqoZWIr!ptzBY)l0mU9YIe`5TBlFD|p@e zQhqn&DCuec%MDRS$p>~Dk!hpWLPEmguQu^+T5rczB7Yo`p`IGmqGo6OeBYM!ILXp2 z@8_#G@3TtM%>A8yXCFTJoma43j|4Jc0X;Fq;lMsK<5E`JXm z`D66`m}O10a*~VKLTdhr3IIBFe5{TSc>(BH_H_6u91H!nC8`B~oWL$0_d|&Ox~tUy zoabw@|HmC3H6FaY^SI6AOWlbn79j%`t5eD7(dZ|m8l<&e+F7E%R?L)rMr`Mi)z!Sc zOt&RUpKo*Ig#wo)`k+&Y6CHokL%vUNV@rOkaZSJW^ktkYM2&2YWe1i~4@|N#+mU9u zy^iZOrl9t@eCRlR5$ zvZ^`hILYJ8AES+1tTn;}3Tpag8~CTEiR`0}x<7p)hyTyUy)OpgnY%2X{rUw$D?1u7 zMw-=GQG>eu!`r!|F&~&%)g}hYEpd{h1UpEFMVI3 zGV_q7x_OdEhL<&@c$$v(U##ptao3gHBhd6IA|I2KJ9$TI={;r`kf<1ErixIj@AHMn zsoUuN;gWdDOWz;-#zMt6H0)lgu{zRA?^G=9k>IX04H+_F!p{zLczHMk{)?Q#K}hI9 z($`q4rsLdz_AI1Ub!yb$)pYaxf4{o{X5g1PXJuhcY1co0uI55UJyVS?8C(p~trH5q zMkDI<-s^qZV)Fd@UsgrEASW{CczATaxK;MQ3gH@+2yXYxRDNuUo1BUDkY>$GS?iaz zd5}FCd*$^V<^BD3$UAp?rn6*xqJpWZ6#nQt;#ji zL7eMc|E0sv!@=^urGIODaNzrm3#0JyR&_$uU_a-a0U3^e`cT0ZGa5(=-C7|Y$=d+f z`~i;>jklDQRlSJN=n z;@x>v>YW*U?`1T(`|E5l_49!HVM@UFpA|RhZezw*$0%~|P zy1!+ATtcv=o}`qiHPU533E7FhsOjX}Jh}ZV-ChEtQ7!J8f)OQQtAvnDF-U3>r8)-i zx8n)>Of0u%b!=2xES7#I%3dz*iq_699F>nXYM`0!UyKf?P~D=kx>{gCuS$eQ^zr)v z_N9rbefeibfiIeEfgM*oX|B`0aDdkp667-6rdE8%C|4 zL^5I`tA!nIm_eNR`?UqQ{-vs;gD39ZEgH@7)zw-O(%rKc>^}4auW@803m-l@BnThr z9k;CtcDKh7{U^i{%ssU+JTA-?WpGWpq3y@1k#qxxM&3{T-j$9s@tF&C_jrZxS)MQ& z(VII%@!kBX22&A=1SJ^DnSXsKQxS7Zz_+S5A^aP@R#>P$NKtZyf}VFORO-l~ z-(gtoM>o;mrVD&q;K`huu0iY5l4+vg@U}RD7?OWu1UxtxU6SkR0BZKGum3F4m+%W9 zR83%1j$4BG*vc>+-N(l|F5H0Fpa}^Ycl)WgQBiK&V3f44XleZkzQJzomW*YX;<8zy zIEw_Y=s8BMUu{e0aj#uxTF2&JFSqyd&c%T<0#*6tCb9VE=Sa@vvYY=>)iV@JvW(R? zZ&KlL955m>xo~9p^GE)I-f>*HZE$(g%VEQ_*21H(?EOdLR;ReKN0HYf>~WaWOiPTq z5ly$Y)#LDK%Y838-gZjub@*M~3v=HZU}zpE&OiUr3^L#)@s@5p&ek1QxW}O!p!HTF zwXFHoVf<0R*Z3omklLNe8)xrIPahW@-+~Fa``hYz!ML%(nU!8Z%W=I;3AB%14JB!W z1>{g&?Bc~tMs=6;8vu8zE5qorQ72>Qs1ad)Mnk|?zC#1H8g&MPKx|x6GE=om+HVr> zZqJ{Wy1!!jWskY8>GL;50uTGAkoQ-33qN046YEQ-LU)RifuzAyb(F{w-*GjE_GF_EEvGzV47Js(2-x(xWb(tWTWV!AOl zm%@wf@UU zteRAs8r2tpegu|%|J5^Ew!F!AJajC{!tJNM5x4%HII?Qb%Gr!R|3{fXT@M?-$p=a5 z_YUXQBnUH6lj?w@zEj>)-5beJdKcT2036=YXoIS{41jcq23tYsyNTY^m0stXA?+Y? z1%8qE8n4YLDkrV+@%3*{?d6wFmx)qBLriddyVGT%+xrT(Z<_W-tk9E&^a?UEnx&GN z(%tRqXa*~3j!%3011|cpw~ZP(8$;d;_Rx}uWx+`CA>Dypy<2~Rng~*5lDMT3ji`@r zSSA@if9KCyatPV^c$}SNJSS<2_+4J*I6Z(rZH@0x%W7>5Rkzo1r6?9ZL)h=YN+ms2 zWu0O%VD@f(!~Uo?kfH8D)#E6iW4eJ_eWsbLpL&!FaHcY)H(T@D-h9n?XLTsdUIOJu z^uAo4HeV>m?j@A3)^j3+s!YNgu7-fEl_YBu5XAY?q)vFV)j+M=-PGo?SE5<(Nj#Hg zh&fjjk#7oYMPr9*bNDLP!(9;Wn0P2>$rYcUZa&0=_tAg&dfTE`!@3+%=6izpFgMKp zDKp}ZuMC_q`l%w&B&~=EN)*8sZQ_d$f9PTk?tqr;ZDmuimg776`T)y>>Doq4bJcoP z=`QT+WDSNMovaHT3OveqRpC?{s#K>})jvu%${}!*Am$ES{y9|h7h}akc(K5h(pk?f zPYpuH;*(y#3Oyr^#LP?5x3YR%fM3bN-xaC>kF=>Vle`7q90@-oFx)%ClF{VsaQ60M zbYV@bmGm_y3bJ(78oTk6h&6Lek>2m}Ii`ihyM7hpiW~?sNbj=htC{g|A9ay!I1LGT z_6<w-Fa(zgiL;>eH=9Wu9rBdF2^ z;lH-Dt1)4HsJyas5Pz-098c%uE3@y79^H^t<$+)=)Tt*|E)-0+S5vGMw#Ta++Iwk6 z<#Xl3_F}Hgw!Mq7)=R^?jd||nwg*aeH!SbbxJ7Fa#N+-ThUTx>kBc-X$zvqhA}754u#gjPYRt-7k&KMLeg4s{tFR@q9`;llVo zV{^+L|C>MKWK8n-=2wBGCO*5AqRSxp7WqCf2KI<<& z8)H4AO?dG`q`TcSo57ru>(kdm)gzK#{~Vy`=h9oAzD4p^ETynmuiJZqHTs?G&wbZd`=iSP0*#`0N~_?baG6by7x z0sH28f+}bH#>-KVS4k+~Zu)nN`q8LHv3ROB>YH+fuUiqTI-~gcv|SC{l(jg3O!&^y z?cwwO*mK}?;Wenh0iW{YLc)Fps9Av_g|MqzIg!O?NXF8egzh)ik^JK5p|Ad7zI-{K zMQ63DmB~T~QMg2*3^89ga}trDy2bN_?<5K zajZ!wxmY7)oBCY-@?4I>vc(FDzu2Hw^xBL5d%ppZ;>yLvhz*k$?&)L^MI?rhCknRX}?2dAjlA2xmdRwBWnt&6;eHkKu_{6vby*tMDgpo zGJ?}l*mKwvCC*FVFHnIg?$qu_`7~B%{3UI1=Uy9kBY%c)QRaoKA-(zXP7-bie`Tl`KW`z~cqNoAqgpHS%$g30)l@D8)Pv$X^qy`}fJ6RO=nb}P zl2~?p9{iw?gCgUc%_k~N0j+ei+C^7|zyWD)#bt!`v0LHEJe-8V{o?HWFDYG^4WZV0-b{nJ2c$sZ7H^vee_8U%wtr>PJgis2 zSZ97SrA90V7g1116~yUpY<#5G4V#xNDOcnJ4xj@+XbMxKomWX3j$2gKA`B_mH9CS# zGEanr%kK@nYp)hS_Ho@Yew+Lb3C@Py!D%Ud02p${Sd;7G8%(maP=*#rion#-?^!lz*cy{N@zKy zOHW$XhV5_q%gTRXO}vFFFy|4r+KKa@qy$^kHU-&WAqk}bVVX9=3%_bgjAsuhPe2E*ETg zne33R*q@S8Uemvn{Pe~}ySGIO z2gQn5nheFXGzqRI8fmjT6yP19YRbBm3D_yELs z?BmN1Wc6%Q?uenb4mI)fnm2$G_JXd?4<~O?8l01+W|;a&=uyRdVMR~EUqtWc7v7CE z6$ilN>clLE&8-oRP~H5v4~xm52#_+n!wHnmRgFXLXG)LD)S(%w?n5qUh z`!XQ^)EmrNLxZw^s2;tb_aCY!C;@(_|6ZVVq#8dS6tB;N2FDJ3-X%7NCu9Ywn?^et zbqW-lUq1=ap}YY^mkVZE^!3EZakj6{w+p=Ap`gAL$+o7vIR}c_fu$U-N*~_OcZiyqXfte^SUbHTW{woTxA8GS_(%jSK%oh2BRHKmNVI+de6X z`#qz96s1iDKJ~%SLrp%#)fm=IdELXA(C zOOCIOdAjfX>>OZh>~&8l!Ea-Yqt+w|(*g8O(@r=0$Fs-Y*Jn+j%qoK*8T6epmqBUG ztbD9h$%FV-&jFFSEJ9|Q83zsya6pX7Hb*T|?TR^%iXtjh82)-&dn?Wl*vc}x`LL*b z^vpBd;MQ=d#c$oD5N`V#`07$|Rr@BOK8@+Yc03T>lP#Bk%Au`{w$O6V;Yjl0$y5u-vxb@p&?PFZ9W5(?QOlRbvjPC%q>$_^&$SDw3@Y`5^v> z@TVUj5H+2ju0MeBKXENRV}&OCjEDdXfY89&qiaI0TC7+PR{R?rkIbJ{^C2hGl(4#P2W_eU_*t6iR+PD`of-;X68%oJzAO70!`z>ZzuR6f+- z#l*xK!k@9JgC9*({IWUq1(ZmH0VPp%;}cI$bv!wE1~N|_M%BXDkiVAsR?dRKy< z*2q^-aL%%y7L&DKhr^DpDp`L)Ur&XIs=hORC{I=m{@X>n#Ln2^@Pgg@v_1L~yeL~4 z%Sc-rqmsq_OObT`L-m1G1h)vE8ra9`A`Y~<$<=$Sp^n1X+dykj{~LNsYdg6p*Yijb zO_-19*_CZz%2?J>zLu;~5{-S(bS}eFDDCAgB|9}lGU(x)X#m>SxU1#F`MkKyN%e3w zwe+iRtlPdaBKh8}@z|XPw2;-Ii-_xE&)rprR%1J9B6qMlfmh*D9-YC`t+4~O9@nP`%L754T7SrvEj#|Y zZQ)3^XsnLz-2#@}O`|_cFw(`n*n7FDMwXel{V>KF`D+8Oc`U@vE_Z;%8`kgXWN8CJ ze7?^86J?A81HAN1Ik>o_ka!<;+kTj zD-@~<$Qg7oIX1Kak-K$d^RsiX>3aKJSg4pXgk>6)8M&;2i&V@8-Jbo5+b`%cj+J%k zGn|ErdKj(3nr?r2k54&AN+*Vy{|2>zOCF(NEgSXCnL(Fh`ZfP{q@sS}=GIpvkZ}Hn4u>-zdc7y=wU;$;E82l;lRm@}1iQt7HqYE^8E3IOL zPe%7Cim-rixl2v?9q}YVa@mB1U`rg)J1Zu1_(@K&XC?5N$rn-&z}Z_L^v`oo91p+3 zb>$4c0yOz2*ripj*1YZOvbv_JFOvek4H`8y{Hjt>3|R}!t^TaeAw0^9^57Y^O3OO% zbK238Yif_D5+Rv_$+4LAd$^^vxIP(H%Yxob;e%uy%CHT>Jzrh*6O-NRqQ)rs^%7+Glv^_2V8J_9%Wlx_ zIpF0Dy^*QRkYRIOw?G^}Z-N5aQ#t}PUCdU{$+ zM-|P*%$^`0@azXm4JRcft^Wn{z*7ZSvh|Sgec!3?C{(8o(nMX$_>!M(;9}0EVHZ#d zb&~z)nbimdN~CZnez~!1DB?Kw;=SUK=mDVjYDq8bo@uCy4+0t+)(h^SmN$9exGF7 zCd<=3S}*SNWOs7*bpBqOFvwY&k^mt{8JcK=RMfpMgxL$NPah!rT+E#E_9TEsWH|91 zn`zv{kK8% zt^EL!?!fXSGr6XX^CHIUnsUAt$1NaVyv)@5k(C0}i%o++mJ`6=$?=*9;WKum8C&8vZ6&(JIHI4ZU1|AHDX;(}5%VLv)U7AaBJL%s;J3WB+gjXTOzqT_GZ zIOl#eTs^!Z&*w?0${=d6)kImY+SmX32mUkp2?mVU?aEwKV;Q~RocsH(I&8q@_O(qi z8~n9Uhcb*gT1*E%cw}+nO41Vs6ys%OloJc$e@Is7O zM+mWarOW6GRYBA7dl5(aUREh>W2veLEWHX(hI+Z--LT|b)MOFL*El1*k0;L%h(I9QsN-?|t}WsR zO7Pn*qMMJ@V5Wt)!i#6XbYy5;pJKc>bplfIux|J6rlXp2sQ%++wa0rY4RjeQLPf7$ z|5BodG!H3U4WF;umBa4}*N^Q;g@Oo)1>Z;Q7Q7H1zC)Z>VZwvip%akI$^>31CUmT! zpQJ4l%ddYl^BhKt*c3%RXnAYbutK=3c${DPz@{Z+%n2kEaid7^Jepls%g;d}+UUz= z^C`6Ouqtf2Q?2wD{Q%Cx5?6aBYp8dcB}38R<01LrSf1?d5m3-ey!D=|bB>b>9ebPd zj8p`}9Boz+dD&Ou#9tb2YnfQ^!{n>;EE?Y1H;YnLBeR|E;vb*eA)aCb%IlL za31{M{%sGPt3%9W8QY)&6(B;8yU$WQAQ-h5>eDL>kgQ+?1S5XYY7yf1 zgL}C&cHaxij~pQxrP71H%F_^T4z?b=3+k_$`@%qs?_793(bx`hC4yG%b)Sf- zysQ%CF1f*WoRT<^31R?-eVU75Q48a*|JtD#{Kierl)ZEDe^W+^;r-3<*+d~q(tijxnrQ1q^U0&(J&w^o@ogwF7Dm&~PcO;q{?mRAUZ} z+C!te-DfCP#ND8;18(>Kv;;(LL_&h|;8(tle~aj2XO)jx>-P!UQwsS#{&?NDfP+Uj zqFQ7&W!maMPFY5{EsnmW`(FWq=7mD8DRpFoF1CAQzo7f7nUT6W?Ch&!8ma#39YOdoPwQZ?Oh!Ha? z<;slB%-;?{6_3MUj;_F&EQ#T-&w5_zTU zlJ{3{Dk_VD2URN91d1>i1vKH;)FPO1_0)yg528QjswBL3e@xy9VaQ+?I^pXRKANKI zj%R;!)xI~A{r3jEDE&Q0nRSB@&m*V6a%@!NU4d2`Ji+8pQ_^Kl$yEg8U@&!@ehYI} z%?&Hj*}^H$<9k>-8hNbIKZRtZyVnqtwL50Qwx#mT-8!Q0G^=0s|yx9PDuJvqw7 zqOqMQMlYKIafjd8dDKeq%)v&lK9dX7*2Q)6U7Q}=q1;Q{b>d@PNentQ=04#3de1o| zGV>p`=v6uS{4VGl?%h0j)~%{ff!c+f{IgLl=>g{}yo@)0(?)GAU<6bx7@2~Nm=hJ+ zO=>0^_6pRXM%q44RM@bg9dQ}=L=2zQ;WcVVkyrou|) zxKXamWMXm+%PTiB`r@hW@x+4Y!QlnQF=<4j(A0yAG zm-d%=E;CF+bHPA`M~H|2!Fm;ud`@S3;!$MhTIrd#qQUe9z!dIu6ukg9!%kv)F}40u zv6fn8V>9v1k>mq){^jqIuN2WLItk9s5I_48=v9t6RE3fsH?ZSWwf?-sVhuxCoK`(r z>Rg9!o)(80M|~)35`MW&V)zIhs#{NMr0K!lBqGm=r0xnmv5^XXr3RH6bZfIr$a=sv zCmbhcyA(?J=wWntTA5ButM|v4V?7Ign=Q-lJ-lGC)~0~uO>!zD-*}LaX)Xixr$nmr}YX{h8J$szC zX#hPJ+Wt!*-?2)KpwpYo(yN@?-}U%I)1|wiDF=De9Gy-(^qc)XQ7zF#H8xiHih1qX zwA~xxt)WAkn@!8Ld!gj-SocvxTSe)(t|=6{(y(<_DkE6Kert#5Z?5g^ln1xFOG{X$ z1Yr4bFEaLXmMA;n&)P*PHOF3rMu#2m%~xjW1iomG8?5rFUD6kn?JDpqmx`+WXVm+i z{)O_OV_i8|?_9{A^qms@#sA@4ga;2RsF`!;zW_41lUGZYwfXjo*D+)u>i>8D#14W^!={{sK?wT-k&H6FbDf1mXlrT_o{ literal 0 HcmV?d00001 diff --git a/docs/vflow_logo.svg b/docs/vflow_logo.svg new file mode 100644 index 0000000..ac7d635 --- /dev/null +++ b/docs/vflow_logo.svg @@ -0,0 +1 @@ +vflow diff --git a/docs/vflow_logo_dark.png b/docs/vflow_logo_dark.png new file mode 100644 index 0000000000000000000000000000000000000000..114a7453c817636fcf0e0efe3f2eced585259213 GIT binary patch literal 34926 zcmdSBi93|-8#i7_o>J;bl88zngtAP=mV_iB%h)GLBl|Xnv89q0OA!)dEz693Z7ide zkg;U!OWB68#n@)X@LnUH_xt__zn|mic#g+?-`9S9&hz}7pVKpgo7xBV9pATO$BqL! z*RI~$v151PjvYJ8|Jeh6<90b13I4O!{hGPgjvZWIS^sv%N^tS-*zxqF&Q*0|-=yh* zy-8>612;Te)#ZHCF6>gj9EnK~JZya9A%~XC+tl%bJK_Wy;SY;&h51rgPzl6j>X87xBtjo~G#s^~<+4c)*p zv3}?`j)~|m=q(1p;EPun!YyVaVTQ#9_+aveGIwAf(_iRdE;Hkd4wvy9+usk+2E88^ zt}swG$G*Lg&y5prxr$m9K_7^a{QLOG9mEJF#*ZK74=J7wZu`D#^VgT}sYffTx>(U{ zJ0y!e%A%Bt*ZX9b^*b1xn~xvt+Z#Q4ZRlzR3Orn?Do*F2Fj-r?W5;FoRh@-P(uHa> z_o$9hBArjs8#^~2R7&YYf5|P=_V)-#cbl1eBSs(i*z3iVLB}9QP^)$8p$rCRAV0hu z;b-ezgqZQOy~L1P8S;3HR;#on_}NeAJs&3tF6~5eZML=}R7VI&ot~b&pV2_73gm{* z^842a^&Utrs&(f%+G%!SS#ap;5HeV8yl2%~CtBH}WpE^E@fh#{`#3hpSdlBU2-T*rib!-@4Beig@uZ!H=P}x&5jER+tf0NB>r5 zDI>(+EumZWE?y8>uc4uCCQN8ahYuf?yICw>ysJtvFeR=pD=f#$MH_6gxZA8x1XO5q zl&y&9Gm2tQ(-B!wgMMjDTXcK`LFbmcute$54Y)u7vC_oVUoDU5{OYZ2rnwPf_vS}v ztLQVehs_wCE!_paBE6;$Gp~^o%|z%vJa5emkDQo_-}twAszYⅇx?F4oYi*v^lM{ z=EG{8t-HU{$;FVO{#C0SVovB=&M!SbJIuGccSJ;pzvk7F=onct2y*Dk@wzYlWi^!YV{=NE+0W>BJ{8h(o7eNM zs!TI{-+M1Np1^xUCCGmvf{BO5mp68=V7FR48+WGRcXHX>{EhCHkc;9LHpe2c^&Y&C zWDW+8-v&5Mib&VZR#P9hqae+Y{D*%A56j7jTPGJ0;?G1GM%@>8P~hnBE5*PaHcT*o zFQC{faJd%0Oz8D{nD_=>Qz_5+sIURwTkKE}V(*+vJh0i|>&J*;Uf;UGOjonTJMZ<5 zsp$x23EB;H*BT>UyeB9beiv3JN2!Lj+y z?r|Yrxn*e;I}N`_>1q@z<-*DhIh%(9z0$|cHeXm2l1>VQMsfO7l=CszXdixl{Cha< z9+84gmF{01*_x`@N00-VTgqc&#*H+K-q#@?D*MmJrHyaeo zI~hJS9cn1)c6HH@-Eh%rOgS}#ee=YeJ%utQpIMnsOSVWb+Ti=m5o zyMtpOLfHdjp>6xdlK%Tl_-Q)XH)KH4P1)dJUY7)!{lH)BG1#YmtcGO0*wUED)iKDU zi4Y_2*?M%6%yCxa zr{s|LZ$FaM$dW8W`?4~*7L%Pdu=ZLLJ*X`N`cK)Zr?2v2q#Za)hC{=e?7 z4ZT0(CT4Ix9x*K)bH?utdhp%$vq)-0&9%WBsl8aR#JS0ks_o&Qu+L@Mp)RP2x`}`f zd9^lT&NQe~@Fo9^^g?Ejz(RP$XvA)8-a@X5z6xsj>eg0zKTbIlMGddf9lV;_E5RLj zRRtyZe0wjg=&K2C>3jXyLL3XinU+iaJ-pi+=1wSreNW^twjASQcq~<$|FgX*Vl7{# z|M=baf8WSClhr}JcR>!AaPV{4cI}K`E^qHSzmW%y_{-Q#M5cf<9zyYp=GlJp zg^&XLwA+zIv-hYHzq<9$khT>7XS(px*~Ha_li{ip{0!dNA;lpQKU-fwPx6>{8=pX(>t3u;iFdCMX$5@_wjt+PhKoX^Z3R1N#TwW3x(}f zxo2}M?}E z3VH2SE#Wn)h08qPu;vTK*lf?D^gQ!oa<$?~3I7VWY0XLD*A$Nl-t7Tfu8Bc429FsS z(@B@PYeaOSA9dKZx^17)a^E!Olj;kt-xk7|ExoaupM0&CN^4!*^>)c0WB!6Or< zBjW$e9r!$v*YELud-$b44$h$cj8c`B6anDji2O?12mJirx6Ssm=>a#vm$$*rw7*%U zQF7;RZ;wg0x;5y^;GqeJ9w%^TUPBSn5nB^ByA^#VYB~JC=z$PoM-rF~CA>&x>tNm1 zu{pMpP|epG0`A7HzJMGkTrzU|yJ)xczOm*9!-+7<_5yKi$+!0HDQW^#bM_NXoftQ{ z4dMSOV2XIrxjoPqp1i}b2L7Fz3I#Vx&+kX1WD$(E}_agd$+#HC9mNPegJw@04x@Qh-+^<%eq z$j;~YZJ07(7f1>J^EiOJ#&mFpSh}CR^RI97Mjd{qmHzv*x^?iR+m0h`AB&dqB1#C~bW_LO5BJEr7daodGSWFPl*8Y3);l-TE!i$Zy zB|l`O*f#tY@~V}hnxxmgfFbr^*=Vn;7q-9PxR7Zc179WQe(a9Q@w)2q-|y$0*Q98n z!I9VJ2*sU(?}`6@+g=>Oud>Tf@Q6@(9dqir=GJMS26t^exyW*qcU`ejE>VLbU?{mg z$%@n-ftT<_@;PojtnAhK?LGz`$^@0HrK`c&rjV5VhRWNoLr`rs*Mr(pij?hKqR)io z{O>2#AJ>h+BYX&tbsq5`W*c~Z=thsenoakw0?m1SlQu;>C2kw6ar}#!p*Z*-gk(!M z*clBHU%Kq}2>g;GhECP~Y8m`kc@rZ94cj_yZ^)rO@7ErxIVDV#QPx@&w+{A4W$Drl zaqzmC?HgsC|9!ur*-Ss^Q%ah$U2qza3-jNa@IGna9^+;7IK@KMypL{gZOVmt^Mx18 zT>JE{9I6JQZOiO2*lpMD(Y^)MEex+-MFvg$w|r_t-BFFw%O!FJ;VeUz4d(q{4`eo=fs!r34~aS860z>^3m3k1xgh*$2UkXmdF)e z@h^cYd2?*dmY#%|sT$*@n>ThBg|>^FS3` zn}YAs3t-*Cnfged|JzPcqe9^>h8q~7%J$OO$rT!~|NrcNeo#T6mi!F) zkY-y!6}6fSV%-Sf796gh_n^$|dKslue(+-RxB)P&5oN*C=8XMz(W2QEoM#U|X()Kl z{T28;{h`0~2c9kWKjeKb(mPJ!TA5o9>=iI@Q+g@o8RD|829d6v;bZ8+e16lW)`Lap zk-S>*$GB6*)=Q!c+=3HYKVIPhoA_WodVS|RrC|TOs&z9wHDIo`zY<#Y5pVM=%twPh zx5su-0o;pU2Y-inUN2DTm+zql+6_%b&Pd}&Rys8!U8*u*wq5LXL6S9bMs8}A-}T}& z1~dZ(NE@Td=QiR?Whv_nF2gx)x^Ay8l8X9z{tk-#rgjC-og}oUlVDY&W`CKD(**q6 zL4Twq?Pu2e*puv!8i(r3ESE2GonKnt(S2&>gs(zSyi(Xu61jWu-t@sW8CC5XwFw#N z@Qsh>M{C8FcU`RRs+hN*%&Vp_HQZ!9)|7|33-Ep@c$(LxQph4BkNo#|J(z`3o{tuW z=+5%xllY~P$lvaP6z;ON99%!V>qnP^ojpCcfCQ=W{8e0cfik1Hgr~j=^Y0||$=YUg zmvy`T$(S&wFxlv_O64mkvPS{2@AY6kvR}U)R`+g$NeR3fk}d|VvtES{wQ=g$Jt|et z9E2?1LWy4{Ya?UHFU@nK>CJW=V^ZucaW?d|ACi$ID?{%jj0SI|?mbE2ak zTn!$|)6|jpQJq&|#48iXzOrqiTwE z2vT!XNvBI8+0>ob#uI&E?+f6DLiL;ieM1K1M}dFz7NJX{W~JeQDNy0p@s^9rD;ibm zs`|VZtKns?{6g=EIYth*)Aj)q$*IeK<;zl?-om)?6zUF>@$s7HQz*n6&;UZO3mRVh zHJblqpKhs}GZw0SVRA?lftCHN0m6w_=sS|8$wju{1Y^CfDB=|tM3w}sJ-I9yuE?7h z5J9read)3byo=%;aAo(wR&a^8W*|k!d4aF9vRlVC&ZdC4Z!Bi8XLP)-G$2T->`%f37t(aQ6W>CMz@HLf zQz$o6NV=z7Vfj1wHDmJA*=z9nlklkcd_3?v4G`?cI*01qq&^i@-DL-EGt#uX;lOt? zi94Mn#14#+Ifl3KTgmdn!#|qi8sDrsBW*BSP}^@oW|~5OQn}OQ8aGaLiA+&$ z^D{i6(vugaM??2F;zJpEhGcf*(5fewT2IJsPK|l2eF<1f>yq zFuafFQyRO5JEsVMje{Zf7>58V_3Wj4midbrwv-?AU3M*5F^Bm!@mV)z?B0agI5dNKI^?X=_I90{FJiWaco zcL^vpP3g^dl|$x+n7CPG6ZtS>`7n{c?WWb3WB3ewlnR9Ik?NiN$#>_6vc0DKXE}cR z(5^bHXn*RqB;nPipQqdP2XyBrN4dlosjQH;F2e-mOGX5_s#0U%HYGs56IY^zzekp^jMGb>}5{+#D1ht>w-%_LV)1Cb9My= z2$;rM$ zMiyw5W>V$i)-`&ejYOGF1#HeK8{plMZR_Y(>U;tdAff}dR+P!T-yT0QPdfX(jEOuG zK5LAKmspVrvZbe+L3p{u?%Z609ygod(1;(TM)x{ooN^OK7CB}Zc`@>SHyPB3tF*i- z_>$1*i8xvKV6mSh>!TH%R1OEeC~fIH@47IF`=qvfgieqN=9FCj!xho9ZyxbAE$a7?v|@?uFn&XsLgqoRMWWYZDQL#UXf-* z=w?L<cVdAxC^&u$ zzw&QcTU{PSB)WLdlJWLiA5D8UF z4R4Q6n~G|2=DFHd0_LA5lcb`jvUA~e)uk$l!0_uB;mW3F0C_DnNz`Sv_+IT6V)88e zvL(gkJvx0Ha6~MAV&IZ$6YwC>JvzZPv{#e<1?}~d9nEJnG0@^ALrkNOgY(FJ4V*~K zfV=`N_z_Nopit;I%r?8R@;OZ(cnGynvBKq8a3uUAa;R*L->nZab9dJxGK)JGIQYat zIwO&?_hs#{8Vhhdc}HLrvi0(H-s#X~7j2`Se3+XsDW*)t<(IEVz#T|b*F|7}gJeG3 zp(vK2M`}9`ymvLtQ!NXueMPL{bidu~61G1=ua`rvXXiA4-x_^IC zpfUaC&ZX)m^OU|W0)B(Ru|})1AMMj#@);l}PLvI(cs7OXIZD@@`I5+05OLEwS;42H z`Ms2U;d<831g;3Z0JpR~ovMizcHAL2H?D9!D>?VEI@xv+v#g3D7`x7bFAbWM(GEyJ zzFqdiJ*y`uBNv^@7S0@i{@|Q}kho7FnrY4B4HSX46Uk+~uWu8_@*v9IwOD2l2lgRI z<0upX_!#P-Bqm=_?p-aa{M13)V>&{=iBSCDxR5a;SGY{jbvk`qAn`*RlLb#$Qt{L2rV)rhu+Tq9-=3oFL zzU!}{UkGHtLIMEtKh0M~I$DNXw?1U#f9vZA7tI~Lwa&jbD5=;-o_Y;UF zDU5|F4kDu4>MXv_mD}d8wEncS0p@b4yme3K;9Ohf02BGG)g&fB(MPDgfC8++v^Xie05NysTg`D^OkUIJaqwCFLgga_;~`gZf-t zdr6P`1=x6-=xH@8w0~*T{d4q0h#INe(9rt{nB5b%k)}~9?!k5a zkL^qy{(Oo{S~dr4F2xF@O$2kx)93vG^k66=#^d882Rr(^+QGH&u4D3rjtj%CkU~L+ zyL$cdatgF{dz^^S&phUNe(CVp*9G-_63mC?Rew~BF{e^rC%0rreG91fkjp0?Qlf!m zC`3$e>7Cf^TY!E#{LM>kH4NlJa+BfgKU(Ip7~I|Cx}?4rb6C!Lob#8{ybv3KJnCb)}Ie}S&su$ zh$i!jl>Ix;4%G!`sBx9RZqMK;`-?b+uELG=pYOun2TVIf(h!hZYUe)ni{EMU|F#x? zL}Kmi32a4^wLpM%U^VLS0m&niADYSqF34SER`)KPcf&>82=?{8i?+8YdzBD%AnHeT zme2g6E{9gRK*k`^^g#Xx^6GE&$Z@0*cC4y^Vx)GXwy?W7+!6j^>CdO4h8em@=h_ML zM2`CPvAhA|<%Tz_k~DNuK^*{s>$7iX5oa%Pm2^7O*)@?*x*s8CWUtFyScos;%_+HB zwASkSvIUnk7V6V@9?ikQ3Twd$G4u0T_FlR$bWX1DW!E#|^^x&JBVk$oC-E&mDr@!Q zu)PbD8`l#p9yL{Tbo329|HNa|!j}&T!nzuqH|%qiP)Iq`fT-`0J^?Uh zES~}L?KE5J*|Xh8pIa2S*A>yb=XM`AbLcXJoe3kwwOHBkC%BH^x0*3&%5uxi`*Rj` zsLWo^^R`Rre5Xat1(Sz$Cb#%8Z*k!68S*u>#`4>|cQZS`YW5qVZ$-*-SP4({IG+rcY1!0SrU!r5EAWL zgYGpR>Uo17bh|z}{^-==FEB-1_~1G0i_CCr0|s#HS9jfY<@~prFVARKB0Z`Ctchke z4sKvKy)EgpVvr34sUAK)(%GGoBm4Dq0C4j3qjxzHqed1s8bpr94r0w8;$K%i6hqIh ztsnEppe@C)v7VycDYt|+SU!H7!m_6pL7k&2v7mz**E@>371U}Tc7Z=-3Q2dNPD)$c*6|BZXvV+9A1GRlu zQ6N=Wdgsq@Ldm}1yYA19Tc{f0z2ZvKjQL%{Nt6S%k0Nk0ECwcMq&-KN#Hh+jD{3gn zPBcE>|BQz#(#1Ctio5AvYd9BQ&_tu0r|b`L^(4dL;-5U?Ytpmjf*(9&eEU`EMod?_boWR(0icWjmbB0l zq5le!Wf7F*z7WYm-hYswmCqA5X}w8B?JHH4)Gr`Yu$(C81>?X~WDLC{%TVm3hUSk) z;AF3Ijw`+AS-0&T;TSeW#yP-2EWFASq)wit@%F}6EuUjv%tZT(VKMvCCwdx@soV~? zZBLaQ6b!lnX5Tet{cp`=*`s^O+@&SLyXduZE{|vaS>_{6;FS1$6%YtUSxT-eBR{AZjm|RMH#h7n^8>d{!n8C<&TWMV5tX+?OXQ zfs^t)!5n}78VLu(-e*3KIwe1SZ7`XAIOYge6JvRX%R@~ykwndW;J3n5S!pU+Hletr zXZtjAjts%_*jVvD<5rb#lOiRPq;cG=j{|liC7sCrg1_4H3w|NfSP86;FxbwwYH&G) zmjx}GUUMPuw1ZVoJlV-gWpS-OXrQDd3%8QTyJ;89nlA9(@Xmm0lr})xcuCw@4;+uj zD63=6T8xJJ@4_D(z z`+*?!-1ZO+SdJ6ss28bd;x}^X_!!IIVoWwY1JcjmyT}4#!~^<92s0BUei#{Dx9*=+ zR}cK=eq@7e2gAZ1kfXCDm6M7 zsPm3N*2Y%6NVhnTx^gU2Gz^PO1q{#cv?tn`u*O6aMkS{c%{!pq-V#0gt*_>uyYn!E z!djn83VspLlGAv3CTU?wvBk~hm~}d_@OeM2;mc1YyGjL95qcpy1&76rbc9@=J86V? zy1zg4$9YOa41?oycUAu#R4PeDjJ1AJs+Rp1(m z?SSNTPWflG`@Xavxs~Q+x4EZ=C%dv`KMw<6Q!U%|dRf=Uw@*1#{rZGE@H}Uk7Wwes zQR|BoHy6?O!6%alnHn;t!C8YVSe>C0W5tOkLWqR=eiqta)pFf2l$w0`n=$n)MY8A; zFbdv@TKA0MS+YspwhXs9uiG>}LykLm0R^f~$^n8f8E3qi%HlJ5+A$ z?P}Gzd~+@XvLSn{3c1&W<@aig{*tt@K0Aok_CVePV-2b8`8_xZU@XszRzwvpw!tp&?(aC=OI2c%g4+dZb=D2d7O_36K z&TV?$-kY9xoHac=8!??nFY5J+yqk`@f9JeZbA3clI6AjDst98nEvRlAr@ZBPZek0W>2+DtO^t?Iv#t|>FZjCW->!0hyEFhs%Zuv!>%yB_1 zvG9kL7ThrEr_IQ{5mdI+0GN??lY`9CC& zA4BphvP-L@**%MBJXkp>AyhHQmSjaZp|5lYmWI+ZVjcZ>4(QOrz;a4`dnK zOI<^TxZR8pYQYxcfPHb+*(DWn$`KKIeHODybN@Jlf`PBy;A z5@XBfXS;TV*)3`1N(`+#Q;Q|$I1gV^y{V!=tCO!Z1ig2#!Rg(GR{LK+;g34lK-So4 z|F$`9$S7L1j>7)ChT@W$tfI?y?l5|$Yf3f%eNTafp?IU%~`TsT<`rzfU*`3Ara zJKrA2Bx1RNTPy*`dHv;|3vc*T3w5nmp{?Qo6PIF&Fi~J(qhP$1o#PBUPz;;|!W8nJ z@|6@fqQ!=;`3x4laoRnl;8_{z!l7GpY$URMx^(8;B}w|KlrUI~JcWx)VG!-*S*SAY zk@p;0bYSaE1qUMZjv-?l+xB!~50JCHC`V8>cte8CwyRsX9~i>>XOBh2IVBb)D@CE- zKr1f(<4m=u%zrl#Ay{EfL4ii(C~bF>xvS7DWvsv@7Q^tMA6OC)yDK=hwyPn48#i3h zdyAB}=s^j(Zv*spO9dgX&ewnEzcn)$QWh{CLFsQL04^*Zn`q40**!Y#ArV|U${T~j za;Qdo5ujf#x{s{39S+yD;B10;Rc6XtSWaWIS=-0P^DcXG_;0z$7ieWBioz*sfbZ^& zS9r=IMP5O0%Z6ZkI&sv=cOa+mx}b{|6z`7?uqYbAt@vD5JL(cOAWxa@7_TRG^D)it z;=KD9Gu0moK73N?IIv*41QGy-8E&|eX>Heaf#2g%CSW)hmz~(}E&N%u|FaO7PEYTj zUm*omv&HT)-8e3oV3P%GBWi`T$lH-{stQ;MYDY6RuoFaVsl?2mfcY{a+(T@*luydX zK{d{D*S8o&X=E00W?ZTv7?QJIC*!i~PYuoUvRu_L{EHQd4(_}MZ%3N`9Uz-CW)Fu4 z&H~32uQ@&(v?@5c?k|lLl?g!)We?`k3WAO^c+H3UOp)M-c3kr`aJdUL0N2&m+uhZj z7WWpaFd7pyqd!>KOP$3PJ+mBmR<};Ef;Mx`3^jC3Z^C6-bki?!=CQl^T{x-{P`3X* z9IigzxL_z{d_$nVo^hWodpEaK*Ijua`jBKqR6VWMP?^Oy0qwbD${D}#+Kg9YO)D{! zXA3~Ri;2K}QIdDVJ=kL!%nH^7z&UNEb0F`1cl(76W|F|doI{Q{!p>KDYQW{R!k@K7 z7xd5^Z(!1{(@!rhhEAM4_UOhK@xr<}ChZoTU2W+Y3o|*NQ}(y9gFsU3)8i(5cXu8$ z$goRq_Uo|o(i>hw^<%X)Kh57qNJ@um-M?#S?gaGoHMt%@`$p0KA}lL$z4f)@%q;HJ zWJ1WlR2)d^gr^lnNDElWD`vUc00OZh4muHs{>)?Sb_>xX-T!TI>Flw&q!&pCDrrfK0AZ)~B3*xb`ZI@$ z@2*CO%dw^5m}R|16_BzSP^%5CBRy&ZLn<5l83TN8TohNQRPwm4smeVE=LKf_p3d&C z()z2@=e4n36C$X5@4SEkqM_|2|F=+$&_&G&$~4s!O~@x2egdTlq#m+B1t&l;-TrTF z>n1_w!?Qa4y+C6#%>kqt)`NYd5$lfuv{I-p0Jq(S-uHl3cnM@ys#zM;4|k6@EpnYj z`uK^5tBz96yGkkd=||V+knMZG_|&o@odY}mEF>^U%{fQ;&5%f&iEyxOwY>5LpTWWf zcJ23YV&cZ8e3HFlScI-^1>UJqJ4^Zs43XkL9S(@c_gByyF;K=931JO{c}$89?3zbN ztH*le3m2vtS@NedS5Ap!;v(Jy4CkD5;RZ|ZqrryT*l$xkwnB=?=@dMXL`#B7c(-2j z{P?T;D`_Cm*}YhCHK9SxV6O5C;1}pT>ug5Sr4XF#v_i#@+byz^))-+w8l;9n8LM^|Ysz9B?Q>W%$ zpIo`p$eBT?OjIab9;5v5ihU*xL;SO|Vj{p$m9_wr4V2Wy*cSlZaKUc7q^i}g6vB3Y z_CXsabRsV8Rl*ArUs}~3Q@v^z2WQXnM<|}J!(i1O=}nEMcu<$gz)W}7UD@5)CIA2o@>|Fs7iO-cV)P>iuK-)x?|mwys-mm=Bp z6lil&i$#G*1vXt9`XwKKzGupsQzlZ zmMZ()?&k5-fEl8RPYv{o>pX}r0|&pw0j{tE@AF7$L+xY4P&jn%$H-{8$^Zv4@{kVP zA}jeZUex2*z@*502pCPog_106eR(Ra=nBT(d~5d78ch|-xZM%hnz`!Z3G2Uege{{? z$ZRzsMGY{nytk`S0nNB)i~)Gp{=6h*bD*d(8at?mpAw%U=5d)Ix(|R_l3ZfdpLxz1 zQ7{k(Z@C8GLotJVnZ!L~#m>&dcnv#l*nzya-M#b^Fm5$uFca6?twq#3`YUvK zZgdKj|J;9Fs(CS#OixlWs2luw5Tf~RE-D&<$?%oGk)pft`Xfsme|r2pDun6+`v2Pe zLbzQ`^H!7-%ZPxz_^XpL4bC~ALc`#7-X|)O;u4xzH}&}o`Wi-99g2gYA3L$ zhtB?HalDTuZ-45c98LN{GB?GrxUnM1bdW}=f;JWZO`?<3_Qba&NjTWJ@C)`VJY)GQ z*(*XHxd_hymHUO7l=ZmnESS|dYuTXd7`H45kUyj(DV?o(q=Ruw{u@O{|M{}2N!e>| zP@{vfC(-I`;14~1m`vu8C;h8C%;QcRkTl+TAa~u+Qq{{ZezyE=-uV=Q>%?$EypU46 z5PUxy_ABP=iOJ5knFXZsl;3q08QQ&fwtD^~imTil<3?`Eham>`Dzz{WOSF5+`Qgk2 zsK)E3iLDG6t)#3<)W;z>omkum?mdG?)wbJCFu($151?uju7rRs8JaQ%0-=j1@Rfs1 zw`%(1G1=-JN5vG?O8Lj8e3wDp&}df&-_d5y5#G3yeXbq3pjg3#Y+VYR>-dv>)Aqwn zKwNtlsGZ@e7hZ@-9lz*B3%sTp#QyreLRg2n>CUa`xo^rH7A8NmW|9^rs{o^Jr@`A& z{VQ7ny@CVh1Kxa!_wmeEqCG)7b*6cDSgxvp`!k3cer@(kbZ1Qs!w{<#0!24ICsw^$*2l5b}+ z>|s%pbx0Wy7MSz=)gaL|W`+|Wr5jSX$qle?>z~(xeekGsw9t+txW)pA2XB8T2C?)* zX}O&@5hpw2*{o^UAuF_Utwrbh%lnZd=Z?b=Pooz=5gcW&#WgHhG7$_CDT=q9q}rh35Bh{f0c z>un*SZ)NrRFJ{@RNy{)mEoDZraY?5H(Sw9RVT;S|R%;MwH*_ro$Y&A@|51X=8ub)( z+2`3nhi=sV$|A~YH)nQ9g1x)r8eN^Xo4f1*1*5h!As62z&h!Oojn6WE(asq?2qMt(^{Zx9> zp*WtisRMet@0#d!X-U(NA|WG$xsR)1m4?bqmJf~d%02xTe|Hp?hLqvpl{mk$j1YOj zq_qfKl!+V6h80wHcFZiut4t~Cvf$%QSYH??iLX)OdGdR~uw0+yDcCa8ezK|x_=8va zANoH+w;z?6tf*vLaRjv~qFwv|>j-_RoOZp=&8CODtmlYjf4a<7H7`F~5WI9r$m{;+11V z`nZx@aj<3&8bS+xhOZbTRe zV&B0icOF<3joVB)3jxM{MlnMr;nC<0kq)To={)IR9&zWn9xj~LgAjy z{6V?WptUy23=t^mBtd7a{BXH&poibyixMXIsVr*8NP8s9t@z>DAvt$R>s~TOrDk`` zrQl82EB?b07YQmh0X;ER+E|iv>q!5(&F0i8Q0zRj>T(`Z;G=f^1HFXe($Waz;n6`K zv9b7VT4?#RJMyrPOT_np`$m^puC|=^7iAS1)oLRe}fiN`+luUK-toJM8ur z9lS*2hY2Yax+?!@|JX~rvJyNrmzl_QYfk&T|KRmSyV5E28pgyN5IkxIQEj(;l80KbC5-_| z#~8B6ClmjIR8o3!D|<&0NOEFJj6t6PbcMWPcLaBbude^J?dgZ*Gd2q z0K>NrUb$|uCo^ubl&!{5Kktb950>IKU5PdgR_PTGOi2VDX}A6$1OTlMPza_7I4dRU z17@-1Oxyt#7e^d!53F*$02c`>PH0tOG38VKn@qXXnLu@XUmatKFSh!|q7bFX;;B-F zT52Qw<&BEX0A}J;(6N+^Z`nzgN)TdiIS)G)t3r@Ykqy-0 z<^C=8|RVEKuu6kWvfOfg9u#v&6v(tj!TD2%OIVN=F4+ zWRwG7-O%Bd?kfcgQFXJAgiF;75j}b!fM@QjUG$9dV3b0eY_{aNmun@4zs=MvUO}N$ zSI^Oor{EfCCX0*gV{~WnTO#zQ=<4OOfx)kp69st87}vpes}qXmjPz%(fLdl#M9+EG zVCh)0_1E-HXD)2X^w}-LwpbFfG=lKK`jwTG9{^}midO5h1y$ey5iKw~%6=h7a#bV= zZ=5|nFd1=z;^ChS5P)$IKw>F zd-lQc^$CWj0IFH5G;p4?lLiFO{aKEnN6O@C&bM1JwEuxE_zb+?{FW^qcpFq8z-#BN zjHlcQI0x>&1e{DwYTCT}D4oRn6lJU#D6Q?sFMjD5snBJyA~9C=R|Qf)1fxP^xpG%q z0#hFYRQsLN-vzU<|1q6I3FkbX#REd-f4BN#f^$rAqnJ`1wb`7On2)Av1D>O6EqTr~ zyb20cGZ+nB23&L1YF&^qM%zRJ)Z<|5rOT*EL8ch(@sUrPxi=yAn3*L?gUyXts_scm zcLpT01%es?nzWb>^W`n~qsIRB8O3)#lbJt3Ekgy6eYJD|q3r}EY&Io$lwvtMrv|Zd zv)U9yvI(!KUa6^h@O5nfHiGgg2`MXiwxqd~U-9MOj26uzrDnX*J!p!^e(=u~C6kBB z;ug!tmyTt&!}2Cdz8h|G&GZMBf4G*y2gFnUC^lK|4+ivz=!bd5&k_3BM99X&do2GB zw#5?c)G6Nh^W2{(m=7UX;ULw&+j8E;Vu9)!1_5$Jn*;?yN7YX6rs5@7EupU&H-NGY zWH!~~R%%TqahFG(0trw!Chge>Ird~&V;rcaL^r<|tZz(~n8`i)JQ-wb13{Itb_X}% z!CNY;h%Zi)$^yo_mR}$GWPiB}l48e&eA}A^^_6H7&TeEDeMa4zjL@8gZP{ z>TGdZ(qk2}w4M_!sG&QXY#t5eOS64Wqo~)V;(;l70W~dXRl8gy`A%3)u`c{_GU2He zMuXIZ$yzIM+;0?g1d~QQi3l2LTx$&jn3|frN;@b}MLItW(GbJ|$;GO}p8%CM@48#x)ofGGBZ4(vF4m%4y6ZXwzvkrzjtW zS1q(nmD0`NE*U=4Q?RgyI{$BvadYy#N>k;+|C^h_&@C^{`n1zVhG;`K5oC*4Z?#%>~?o
      ;eehWm zM+)XGuyCw-6URzt5l3ruYKCe|XM4w9yGCUIPKU(6WW7e^r?mjlWA~zLI_1&H6?NnJ z`0wBktQ7J-Ucq|(0I-}7Z3_BZFM(ToBGn^jjDx%_-eYPZaU3uWJHL;NB1dZ9yIQEc z3*iy)|7jA2^|~LbF0j7W5?e_sEk&MKA2$rCpyOk%9LKuYV(j|T&5 zC+QFAC&g0Vvi@oW#@-bCH34z;CwM@tSRvz>e|-#7Jf{4j;s;Z=wX40r%jg3lb8Q@T zN3lh@&)OHD4;hU5*M`ZUSXulg4n!?VteJQNmKP-}?KER(Ld4~c5n=09i6bAV%0Z?z zLp{`DwW)hHA8nMC{442~#j?d2_QFF~W6pk;Cf_-#(5}<~AdcZXO(Z6*wja!3G39#Qix6U zkcUrp9dyb_^bI~OPJUcapm=K_*CJKiE9^0-s$W2=!7)R5x$V4>@wbEhi>=C2CVDTK zF0DBrK(+j|t3A8qcfGnU+`%Vz#Dl=G^}){w{v^j$%}$M{TwVgT>8x-lbxEDg$)R-q z9`dK7kX~*}j@Qm!WMd^s`kZ=+M^@kK5Z)4RHUBMh^jSMLpiO)$i;Z)OFeb1Xsy>CK4 z5*jemf_7yAlzkZTR7NEl?q3uBEM;iYV=-eQfx`S@WU}FY93AeK9+iFYIgkhf=>o~T zys>1y>iSFbEuWPM{cH8xK9>ENwj{|YV zb;2?kOH|t#O>nTPO;NE?Uktq&Wk+8ZqXms;f9!?%K!sE7Jp_d2RzAm{4`*gWn@sU8 zOuTYI(iopV0OO#)z6>r2n1O1|fozLEQn_t8Eja4V-XajPZRGHqaCY{Ra3oqOt15}s zn^I5*P0*)T89xl1rg?VB)nV+^s$=XDfN5XT%C8QHp z|EIj~{%Udy);<=BAksUM14wTN5a}RYQF@gY5f!CMlTIukO+W-xK)TXG?*syZQbZ|{ zP^3yt03q~1Lddt{IrrW_;^QaRaxM0LXU{zI%rmn$0Dc{f7?16SY{3mETeqL!SqRGI zuXIbVfHes)WSP=SOV-V194>S9L97k$Ia%%)j6Msxok+f#Z4RFf84%b!)_7G#2IEsR zU&G5_G4_FFsX<&6X(g1rlcrYHT4VY1uZ$y?@Fc=7v69)6K|k@$7y^Q2Da~J??o{^S z=D#(LwFwwwV?-yGQNc9>=l-e4S4J)oaCGVfGbh25n2pgrjws}UXUdzUQ(X%B=@Cvb3fe2kMxrl2mSFsS$;bd8O_aD2Z&sC`B7O1UKb%;{R+!V-&M%F=-s@bs93eDYrO z!fHbIjcT6=>&@e3qJEMqm{|eOd-a1heH-@X6P=H_%;^xZBRKb^uZyF%G}GKR;6|^E zH~r|hTR?vnyv0Hhr2UNM6h3-e_|kBMc8ZU~R=IaXMi&~4v&~zld5%vV`CM(6*c;3S zYY9&jN0zsU#V3jTNlAv8)YpOOTa%QKzoIFRd##=Mn1?XSUe-Ml$&}KTY3WyR5kjWk z{NL~SYG0*PZBX1;`e#bZ#xt}Q1Q9!hc2}M?2F~Ly>udV4s^}G&da3LV^sk%~{3^cG z^xFiUT`_61jr7;h1ox5Vz4Cymkq9vf#yr;_ZY`rUH#g77yxa_+nFfucAi4VoR)Hc^ z%abXb)&>9Ras8zrcf18>WXF&+yL&d!;OdRnDWF7l@)}Jn)9109voK8458ad@U+fFb zaVkzpk z_(87-ObCA%tj+x&r?f|IM*!cS@1^10j`Zl?jWPLTL8;0qNKididEyYLd8^rf+4L)n z#?ShALC}x33+qLQSjEW~-IBZ`1-MTsUfA?waxF*7nPJ~e!6UPw3F)=xB0en%D;CG+ z(!TA|zI7`Z6bj1e0XN3(b^Ko-mJXExXt`3omPj_koT+#Hi5P{XQ#yB+V-=Bcs<~32 z1b#!lumrCW^eZ`*G<=!;7?z#tW0+IBJ;x^ZW|i}JiTE0Mf5c-#(C;rqmvjz9yz^;l z(+UWYc-0#~nvl}wp!9+i0QsRL5C7l7KC}j?NQ!waB~$P+{RnyJa4NF;&$ig zfNK@WRZE&>jj=plG0xjF(|$vr+!LTQW)z{E)&M0I9{Z$>H33 zN-NnzS8ZFKZw4!cG{G5jrtkeW9KD6`I&4vonNE0zG|m?lUS|KmC z0DUywO-jQ<_7zSy$*4(d=;RCwm?1EOsJAd`x;Am_TXa)QXW?e1*8(YMg3Et7^B>xd zJQPZdDaK`5fH4y5hyKQ31C^8{R>EsTY(BHzvgWu$xww1eqn)#x{T`xbME+!lWNbNr zVIaXY*ZshVClNiB>Laz$AMVOMQ!*RlNQ6Ese)pvDU*Hu7z}r78y)>SFo4hN1y9Y6s zzm!sTe*3TgjIZeEb9{BCe6p3NjzS;G>OdA>K#V3kG(J{0Q>r)s?JO&|*W-gdhWB#+ zy56yr6F4`b_7*(_z3SBL0dwS-6YD&<3%TLyHXVZRyw&4}ISRekaVev{RdQXO`oj-5 z*i3XExs0)3n#KimQ2W4?O+9h01BxV5E#dL;$JO9^&mSjHR96kmXm{Bk20lM$i8*~& zvD!O48sXc}`m|wlZO$5FewNZ^>Y(}QF_=w;q(AH!2{K%5(i(3!G~ZiqJ?OO~_6koF zy=Tu&y(e|X1DZc~zz4+Ry5CA>bnP`T0j=10F%KAg>Su_{Wr;c5Y06`7i0q38;OQYs zS`QgDz>$mrKZESMtvNm^PqMFDsRo>%tUK3cPr4Pzq3^rCO|#TNbF_B+7$4GWeCD8% z#WQThuY)uP>(kVj-bgj~u^o&c_`Oka{JEQ#4eloT*HRXbe62o4UuYNcH+%J8i9rfY zdv7ddX>r!LpiS^5OKS!%2c=VK_LH?q^TF*F{kvp)q_6F{OdB;^duAqYbGh$JRgS-t zpThXG6iI~K>X7~2Nu`cQ&8jg4ls@?}{%_l#`x~47gsw-V$CyeaPf)rBWCC2g%2&fj z>Z!S|o{-L*z{2onL=A7vF}}YV`!`wt_<0_!TuatWtC@K<6*D~cw7tVeI+8Who~TyC zL_uEngovLuRTqNSCJ%u6sk%Gcpc8EboWUS`FUtVKXCYv+AxkTdG72gq2 zsp3&3tN694^t!w1SJp@27C9r8?@@bpp9ERC!a6fwoE~!~h?ghY7Sh|wa#V*J;Id9) z;}&Esr5yi~xb2k@Hf8LO+F;Du2sQmQbgsgAw}iERwxA6sAOYcc!6ffH8tezw3>wXuhcBHQ(;N3WgY=wiK>BvJoUT{QX*2j*_B(0UdYcR!wR#W%OJU+Q?^^S*@EOr!T)WGq{IuQR?B}O6M3LC=l z(j;q!y(qf2zglKl-34Yu*BsBuJyu6l2rNqu7e5gXm%5#D&hAsZ7SdO>EqGMHeRutN z+U1P=D|FD% zxd*T{*~9K%?;H~40;_zl6?zHBD;fspCUt(G6F5%R%a&{~n|tEp_KPcMhLB>8iO>9e znj_;0Rrz67&@vWUNN?wqV9=2~c*dMfUiVC~oo=+2p3Nr-j?G+T=&c)2ghKZh%j?#M zA^FR%Os`DZj1D?%^-hIi6DNy9?^)zJpnAEQ6MuAkNNL}TAlvQLP_N&>eRFU7sX+1sK$A8PFrdq|4kE{B{8(3S|z>hP-y~%kU{F_|i zbNP72oXSMP0Mni?wY%$RguM@1(+fK0&JDglv1~oUwd{(CV_|ZBa@}$}+s^#`710zq z^c_b7(`qIW?CwD1Y$LPIVR7LUhk=z7Il%??qxc8lJ&#R@3ATlwv5#2=va2Jv z+s8J(k;N3O4j;MIJW5f#ymb%W{QLD_ephjM`WeXky4pul1g^y{-(}Uqmi+w*nUC)S zs^pP!&4i%Fo?pri7XqP&vWW*D2|*AYh)*8VtdJ$B?w3X9d$~#?P8tTj8w}j^?TY(nF1x%Pwpip!QX$LBC*-n4q;2d-cq>Hlj@L2 zZ(0sc)3GZ^x#06jvSt$&?Qm4P6`l*K4e=R%G*i=LI;*K$6(RC$s@LpJDKD10$mdJ3 zE}4$$zVnAPL$w#=c=?pKkQW)8be!~dcTc;@E?9u4IaZ{3NCw1%_DMUPrnj-*1>75C z4qF7w`qEPyjD5~0X|<2(X0O9WZ$j6KQGCy#=Vq9@h{YNW#!k67FZvel$Z%==$LaFP zYSX%1;D<@C-WH15iwP`4V$xZs_BlGISaq;L=+FJFgNtBa@&VZpi*kKdxdFfle&u&A`P$qn|fcdG-ns9ug{ugdWePpE2FD`&)XCt>8` zXm^=y{kL5ceTYN#V`H-Lb-_ZFeZssuY6q1nEt6_3eV4dqd>*xNj<1X8%XO58t$v~} za{JUb^%)IbEl9*Ek2KCcJ@nh`DiAmV6yo~Sbq?J=4Ka2KHic!OLw0Jo_-wzS)@-EKR1f~y z@2JX8*1Ej}{h7Nn5B>u#S43DIwjreUi8b=6iBq%61k)bjwf*O5ea_A%J(1hK_PgEX zB}4oL%1Lrmf4_|DH-^M|FCHY)qc(&D6-qsFZxMuaxBT_K3FM-~Xgh}a$1jG7p1gY& zJ^KnddSZolZYoUHy5`CJ!-jNJ#yPo2Jn_Z4(v7{H)zHR>rO0{OYR2XSJQb~2*I>Ro zqAJjnWY{F|K|NgIdyO{g_P|2lgIG1K*9qOmaoy6_O>8d^cr@NQB^DMuk)~17=Mx0>FTA@XF~n2<_nGM%Kf}TGh(dYv=`m~5hVUd|!5?Uz2!Ufh<|Wk! z3o#29!JPfF0a%i5eunphto4P1rs#lQ8?wzN zUhC5H9M7%uYTRht>tK8C%E)Iuq$d3=Uz_L%W{?A3*e-|XWMVoy>(9K*BG`i4WG8B_5`pl93I@xER z`WJSQmO~b)%BjcV{LyM$H3+<1t-u@}`_$rw^woQNXV16Oe#MO5A*>lpNkzd_3GZ}o zH9L|M%##@Y2U@UppF%S>^$Wi_$lTt4si{gN%0_&Qixm?yhX-~Jv1(UFqmk4Eu4g@! zS#N35qXvE(%UmnlZdU1w`5yc7e(T9k>~$`S@S^!E^dS!Cla#Ps*4Fgn+W)%%NA+tL z=b<{EwN35@hiT&4`Mqsc^Brte%;zC?aR#Sy^YQ$A(m(SESDapx_&|QuXxG{e!Nrnp z&lj0zJM>Mz{9?UzQlD~Q()L+1NptqoM&L4IAwMsh3`cd}S*hE?y=^(D1czXPJ2#;@ zPAmi6u)e?LuOBJ|35C(&YPjkp)6LU&rmKk{I{yIP9SY!?j@wWGFInMbG~DJ}J%EGY ze{G#E1r8Li=-d{%lR*J?RPRl_ua}}=)W?n~KZbd$VrFvJa@++FPU3*=nEF1LAYH%9 zR_v@b&G93lGwM9sT8~FRGzm<)jx+nQ5|q!xY;Agt4CgV`{6RIzTPqR%uJ@gXdbX*R zyvb~Y;5e}sbl|WQl8X8`Jf(J+w{ksk{PU*i6dsc|^`^&c{2@MS=B=KA1#1CBLqIpy zzQI}RPVE&sfMlI=)JbJ&n0kND|4BvZ>VU&^Xhv+iFS_sfvY79vR63s0f_{*WYLh8D zsdsAOpKVWzmR|m7zAu_$?y2&aB~<4Hi_Qt@u2(>m?NbPJSpPn%MW8fP!|3)#( zgX(rud_kgCZav_L_M=(tg^a1OuT3k)IlG29#pwZ18CsDZ-XQy+Hf}gzMdu3EOa&J& z<;PccJgO}oil+mVlQ5yNVWROO|}p8r4RnctuS4KmQt+qhBKqtaFo=Wx6GY2xB94 zs9t@&)q=6#M_c{|MqPN)q9D$6K0Ddl&AwnQq*x#P@*Y&W)85i~fySwJawF&D-5cZ^ za>^ny_Ts(&Qs2AZWF`Ybfmx>lPr>1C0h)y1Sh0wxlEBYH+~Tp)I-0{tCsyN4G{W;9 z2fmD?S6_1)zX}*>IF$yy-ttDgnhVP}cJI=n)5`gkFh;%9{J(|`2hNM!7a^9I#gt#cVxJJ{{_4_RoLjZU0GbKKk`tNohbALX#4=QvNk4;0+PnMcc7; zGdc>Z7(FUh;H7rUPyTj!&3Q5OFhuBQjkQnpY~@M7^rWHd@QR+Hb-(}AD6mhB6WMd` z5`(y=>mSSzGAp@awZGVc_0{gA5{&)KHU3IP~e!cYl!#U1!PUA8aU&{!5@r_c~jZHxGYsP%d;bl?y@Or-!IX} zD#^Vo@6_pomTC3Q_cH&pS7n8j)9$2%C7{l%q2C)CO$crU{)T$@*PG!m6Sq>FApAWv zQ+yfL5LV;<`tSmw+4!>gR!PCo#W;rf?72-%YnYF0+sruzhhThtE3n}rZHP)}t^8wy z2Agkry=SN>XZ;C1u3A=mAQ!M1+s7wb4sG)(s^4RSzwhXhSoWzgis0pyvO^x9D_0~z z44b;GKvj`9bn*ImWWMUcKw=pN$vJcI;RT4;jZF|$%78Z=7wDbW3C0)no{pwekxzw| zf>5rN=_QC0vOX}^f>-Z}^~d9THZ3?4c-q?vwsL3lPZD#Nd1quSOB;xSdo4n7F7&wj zdN7l=&Rw+tA}e>h7ALA$ug9FV2%_m?$!{$_q@8@`-|(N>uhdI!d85x&r z1!Ljv{ZkHmWq#(^e1LSG2tV1h7X1qsA_ro*^$0;Ao@Lmb1 z2zUCHQ`s?>^hbA1bn)UT=IDX%#VA&#k8+!ceIXRdbe>Wkh_`khkA*fQ?2%GNH-@faJ_YEY5%ZYjOVOC@_)W_|KHE>buU@?iF7OsXn8v*< z5E+OJ-N_SuG^w+E%n>kW*-XL3l7zXS!d}N4i|GBFtKcXnT@TiT4edU*``lVSzJG1r zU2x?*V;e_y!lmJLB?8~;sA1lc^+Ys^v>Gz&FuLt&h!HyHLSOX+c#62#>kzAg|&*4{f3I+w3xW{6*=FRLU|1!%G`xvtI5{}TXZ;r{mc z9-!DN?D?D#0Wix$U^>f*N=#v|(+!W}wA`y0nGJcZiAT4u*dF=WdM|ZW&ZdO=n}^6x zsrfl!UI+T z7-G7Ys-uG#5s|hY|7ZNIOQLlB%sYOEcTIw^3~4CR%|C@hJ|W^<=yU^%Lsb2!`V?<= z$7K|i2yow$SI3t&6Nr76yPu(-IQh2ANL2DMkbz(GQ zxOg}ViQe*!TXb@L$sq#A9;gVzG%k0G{m-I-I)xI;D%g{clnS%x5y(iW#mZ%-optQ- zN9`+0Kjs=2MsJwjAwAPCU&{0kB4?FboV>*@Sh-S_3;P&iB;(LLnvlo~arrXq`tN=HZ);dFZ$-zfq28LWS7h=6SY z(My6bYv5$4`Fto<1wdd1K1xrZ4)8gRmQ;?$b!k1kq<#X3^JpG8QH>gQVs8|3SvBVR zM=W=yj@e_?$!BPqa$1)y(I%@wNV5gjM-xRvuPf1_UKu$i!*K`C`gG8=Ie*y zDyIVCOEF)*d8a2i)8`U=$16IRXR6qw!O>h=JG~~iH{qcYwRCcs`5lP_Uh~_y7;@ge zEmTeds*iczh4StFfd^Q4CBUU@UTi<&RCe7Z*P6n(bp?U{mht_d&Ydlp+7(rBDa99k zrzJPCjaj2Jl||xhQywu5q&1F=FC4RedlU54MVj%0P?me4lDpf{@HkR+sG)lXuLMpH z6Lq?5+zqTUG_PMOa>sjBByv(yr*TUwN;Bpii-<;PVbVUhtH#aZx)uo_1I8@;TByzA z&OcvSnXQMy7!g|Hp&<2Kx0-`Kr6#KtK)S3^dWAH;i2sDiS5#7B(EApJvyN zgA|KQUfI)A(m-DiJ%f-f)c_HjYNdfQMY+mOO!g;y`sbdHBT)(aZ1hqUOC)bLuY0z2 zf0Zuh$4trF%VaH8w}-!){iq0hEJP;1R7B-8&5Uhg*_eCw)_ZOyN)b+s7ejg4z8Pkn z*(*V4;w`S7qnwO;8-ED_CK-B6MK`3zp56PQZ$*M}KEXOHcDm1Q{BISXt^d2HkBPRr3Qdg9gdnhs!C= zz3hqgV4c~dL6YQn=MY2=u(A#vys0D*(^RBxn(zZ*XvR>lG|%V4FV@-Nk}-D<1i2&v zoXv>Uy$xCLK~W{84ObEdKYj(dYaOLu_AtS&rC%Z-r$i;gFOnW1HfohdIDAOFV&Q>5 z=6bJnhr4loKaW^!owT*7kCCl3-)|ckCh4ZpnII#Zr9Z^v?Da(3e*);U(G6VMAeOK` z$~BS1|NLoh^%lS7F3v1K(>Q&=dQ+FPgeQa!Uf0y6^HHGI;|(8!>@`K(w$PJz^T(0c zwvTa44Ix;6Lq}bo*rd%|o6<$Hlq$^ke4s4pJ;Ed-ZWB^~He;=VXlxrNfMPI@m^euR z0&|NiP7TpPO_|TzPI+mAG{}fOZS896lDJpsub;T;I1(h$CN`>Px7@A{9PV!GEGt-J zS|>LZMPi9kWrP$>n(JcZyz?u~>9FkrM(k4;)tJE&bPmmh<+a6(ApHv(66Oq&JGaK1 zaV8p!C>2FxTkpH{2<6cUOFMSlME5$QHYTBH{Fc3+2(aBvP$$eK58mQgMczSS5}TuE zX)n&mYOCnXCJnky7f1+0%|+am0$<-s&N(iM+6bjbk@PnxSa1!tN~=Nr*$K8BI$i;Z z8-!o4ZHZ5mF;P5WvswOz4M**%ev5F+Pq1Lnim>kYxr&jj@SKOyMhwQ@@*LsKFC+?= zl%>JApF8%5_w1)W#6}7jT{H~l{Brx$KLS0Y10%fuvyhRNVmyCL{)<;xp5t1q*|wGe z%MzvlEfWo+a|M*1L>8QLdgs2d-zi(Ln{Km)_yYrxgR$2qx>{*wL|ulfc72$C7+@4+ zQqm%Y<&&IWqOadq2xX29E8&}Ywkg>~QH<`j`pJdz8L%MzMdxV)}0mxe)JWYDuPPq<-Q6aCN!}+Xo&{< z9(DOejNASGBVUj~5ye$wdrr|N){~|CM}m7Hf)`v`b8dPB05c{x%S9r7V)g1r_U(tm9usyq%!4Z+|5=&a}-H95kV=wT9B@b+Yl_gcV z<8lU)1C2MpBo^II-_bJUEGdd4NZ=k%j}D1m?kw9UtndplWbTu}zI8cG*wulY4gCn(}t}X~teR zhyphlFj(mc#OeRL+FT50X;CPl3_bpQka^?7*ixXJveMN;AZY61btm`)s&p~ze}I8e zue4Z(F!W58TJAjZ+&!eT7SoJY6s1k|{eaY@q86`qDED)vfmochb@J7h#JDbcYMAtw zoZVcpr0vfr86(AgHXB@DAhsTbmjC$CEe}A{m!cN@TriWWVSTE70;X_?s&AM#2ky~l zn@<$*tW-NN3Ghb!iy2qbLfxNc=gSP1r? z-0||^9a09~y<&`_j}VSyHVaGc`lrJTz`;u0CNZkIT~6%mj3~F5>)~R9J2Pj0Y)=?o z&3{X4o)zOv>`XBP)061HZdbWJ*_BY8J`6hW^UVMV6UteQ{~Yo>QtXQ~njtBAle6m% z@Kp>g?(|;p%Mi~I=%O{}Qb(=RQC@s*39Q}Uu(`IpW90#mG(e{&EC+u-0CuI&WBVH5 zyR{4;NJLbKy5^GG0a}LO5;s^e$s`vtL?Lov1yUL5_`-ywnR@!1> zv-Z~U*pqCNE4D4%0Q==NV+Og6|1E5cLTH@oL*}8$DD)Rc}JX$tD z&a0@B&1)iH7gT#oK$}!Xm8@4!uP8@E(d#J1sAJX8n<7d}_VXb) zUHtal^4XN1KOMW1wOz_;#%cY9;t1}x7Weu|&}cvxfc9v7q7$I`U}$fB@=%e7Nhgvq zn^>}1oIW4o4SNYEKKne}B3j@yu{jYZXUF%;g+3?^rnD%qKY>8P`jfd%0+KhoqStFb+QPP}V$l=~^ccDY>`Yc;%tRz#Oy&Ixl2`n`dj zHXL|!B8*jUQrpWZz_ziX_y4(U{nE z1VaO|>=oUN+l4F#eV(#c8AW&`^Kef?vS~6i7yH?1=#K_f6GSM$4mW-eoc8cvB3Y>y=Ju zJbi#9{G^oe22Hs|#baL*hCTe$3fDsCu43Ap&Chj9xv>+KUJ+!#=n9 zgtJmheZF=f2W*Nv z10#I!jcPlup=rr3Cv}HX0mB-bYFV61G6hxNmPdO(E5{er#z#+0-f9lj7w~v+yH)kY zsK2dr@)=dBL*RB;7kxqK1PF{5&72R9#cvHWq*mbnk(9Dppbj1*Zm_Z{)q_J&H zLS;EYJgVb*ml>wg;iv@tDNfOGqPi-MO;+l*`HvDgl0;zVk#gwEB8jN*Mtk_Z;Rv$) zKH=p}pL>zEKKCRMb`8a5z0_)u^#$u9J29aI+c1f{_iWhjSkul3kpjE?$hM!Sr)MNE zkl%K7OQyrdg=4FNn&B6dG#l5(g;6cuO1%{km#JHlFQXMlc8^sxQceN3K##m#AL3h9 z*1dW)VD^tSE?KM-Gq7@*2&UO&6scV_W41vwXxSh#QO0hA|L1gHMfH0yRWPg%*XXdf zo4C)wPWN-_4J5JOxG;%(5aOOrh7o98_%4Vbxj9&!C@O*3I^eOmx0H*{IyI1M?s>N z(9Y*3ZQ?anoe5AHkT~?6TKbg{kM=E2!FSa3WXkiqp!xLHBb>F=AL65X&tveqOC{Wd zhmC`NbGLw=7z#(@&5GVbvKFqx2W-S)F$BD9t^*)j@wTiS_ zq|g+Ks{CRV^9p&F!OM$W)~RII-Bky;>tYoLM-G-xBD^+Rh0chxUUhRVH&t9(;~u65 z7lB*18wt{JJ1SBtNKPzBUlN6lwxEr79r%(c^4oJ-SNPi0=V29*0_s1VXY)65cIpM1 z+vh>>QD!cLGo+WHU^kOGGheb3H}R`>H^y?%>AWqhPJOkZhJzk%tqQtlNPy2IRB==e zu=zx~srl=p%j0YZSdQ2o9fFk#cN6LtD=rGaVm4!92?zWEk#Qy0-FFTOIg?BNhz{zO z>L8(C$Iu?(u-lnM)lNrDh#}sO9cL%f;Y+Q4LM(ec^IA>tii>dXoK$WeExo%Bq0>deXqASSo4Yo*T8MKMxi8oHt*sR^Bnk{w2A)7lJP2^>KIzv7VfI&) z`~t7ohNzbbzL~YtTr0Bcz6W{6`Mw}d+Q#>n|C-(vp*21hMI+dEVAdF(oFEC+V&D87 zU#d2(uJaT@8mx8=3{italL{9&r?K?6Y7?6ez?sa6E{7!kR`ED5cdrn3M8{`z3N-BI z!Hbk{wJj;Nk$w{Or)|8K=*=Nvlq`5Y-Z&*%(dv*9uKX%aHCZpwY<=#StGqKmYPPv; z@bq?q`cN3W|J&n$jwp7}US`O)6r}saJgbO}zVs}5RRh~*(MI{kf*#^Y95GP62*^pV z9n84+b23()!u+l@9mE5LvlZdT&iqTUOV}JzDYKQAaMLT+cXW^nG#uU&kfF3UVrv#r z>pl+zNO08AOK_b-`84h=6VT>EHdAiiRR&4MpJi_r5oyYsZ5gk{Q7z_0A?9epVb~s* zNhDe`esjH%n?L7gwgbL0o43=}+hV&_gw9SYfv^SgivM}Kn;RiKaJz|NFfpcsRuRrL zve4u!P?9mk$b0|MZ#$xxfh31@WHF}&1kONK%te26Uo&}7v>QmtX$GVAz`Ktwh4P82 zMz!(H3|;#vOb!VsYTCJu6bNr@l2<>nv`?>~NR7&6i^})A<8@K`!dIC+vJ@{r+W$QY zXWi$K%+Q@Nb!3D-Kl1`HNT5Ik$Sr7F$V6{Jf+WVD<|R-a`cg`QVUzI!1sW#51Ao!9(Q>-lUYBWlp!FAapThTasF zi3(ys_&lsa%EPJWp2!G3rL#2+x*$zHt4dCDM#M5gV;s&0CI&))_Fz*r4y~xnrb6V} zZ$d#Gig^{QvxHucf$Lu{;Cwq~s_Ms;tt+3vqtKMs>op?KR+YbN@D+@4c#>Kk9X6ZT zIL2<;PI}0atlPg2e^N%|p=TFHKXqJey1%+U+)^B1>G4Zn&Mw9wF#&aypoTmbdY(|) zK&e5Gna%sqOPevWGSUX1jW~QHR5wu>Bpf<@1p**3r0{Nv1^>O)9j$FwH35O;T%aaC z{sCRl0y$;-&trq29jRPAg^%pNy$^`w$|ZAwR8D=@tdZTUwI_X7bM(B;i!B; z-RRJ*)9!KaX%>rRIZ*UtoD;GbTUfG{Sg}OuR*BqBylpB)iv~@!Sgdiue8SCCm)6m5Mbat>S>WAS)!)QuIi3@~+K=eLCA zJR6h)hrj!LJH$fpdH(t<3Jcd+DH=iC zkFSBfv-yKGS}{XU4|{`v&>Ln3t;T{2#C_+)o1R{WChxhGqCyJxEI7-E zW%Q#}r8U@mj4q#RFQeu@!2?Co3ma3ZKORCwmvH1D2oDW?pP0T&Me&W3{r3p!1Ctxq z#gh#%U#)y`P#$Jxd zuaY_Rc{&t#NQEDdZMo_D%MkDzU32gfIFCfQy;Ib{Rm@8-*#RySnv~B$YKg z&WI^B!;7uXFb0I_1NVzf2ap_bjz~sXGh@P+(W}PY4ZX7}!L+KjdM+#$IWf+~qPJR= z;9+sPs3W;-Axaqgz2?G$$zHH)+E2dA{ajKyvJ|PaU{~ksBE@fONb&Si6@NzY{$|-SuJ1g%V?fQ?y=Zv;(~XRUPoLhWQ_Vgw7u-1fJ9cYR zyK1z`uJvOZTXq)c1jkDblUic@6*}l{rr9*&hQ`9S@;D1PIWGyspj7hjT5#TqDc#<; z+~b-PCXC8$Bo5M}to9@Yt#1+O6`gU(o$nt8?fE8U>fxN$Q^siriUW@wN zyDly-Cvm4HSih%cdE$EGmbcAfkQBAQ*qU9JihEH~dNeE007Qo2&m8vOQr&7Us7HZb|^{v^Pbgi#SRL8kDi^^Tpjv2ltz5Mm<>pQ*c(K$hxh&-fo z#bVYIC21Ff3)!}G$%^&I3ne){pf>aZe=<3O0DtzU?r2IEyMRW10a7e6uyx!!ke_f0 zvGFi(kVrGt)yg>BA*^}JP%o^0m-$ztS5NZ{$BrcSilyqRpiNds{n)n!cH#8)YPK#d zC(gQR)7>5FSZ5gp_2AoXzQ5MN91e z(6E;YYhXL90b$!#7lDs=)+S{W3#W|r45`*K^~% z-FQ~R^haqp;U^^HD9lQ$Q_%0T0S<9}f2&FS{y}zVP$p@%bzi%%4qXMGHJL7R_ni0; z{jQEjZ|X~noR?teGcyxfW#K=c4DRyXa8Dv#8%UfhNU%`@GbO4yL3FBGQ7Gj7)l-zw z>JtBZ^Q5uion%}r*wOd%4DVk%MeO=Y*)UTTkC7wLp+ptX4{)iR>Drx*@#fvY&SjKo z$b+<`~qpgZ~ldcjIGHMf-f zyS@3)YFUA>auwG7uySd!{h+^x!@=&FzKMHhz?)3-t|c*VB$}9=D&m#jeV+WHr)u1 zHlMwH}_M`2xs z$=R_M26N5pP&BGnn@v$*&1abRQ(^M+*vFBS tLdWH|j-eizahfYMiLw9R{jlI*r$n0_VwM&8YW@O0*Y!>GDs^r@`+uiQNu&S( literal 0 HcmV?d00001 diff --git a/docs/vflow_logo_dark.svg b/docs/vflow_logo_dark.svg new file mode 100644 index 0000000..db4ba99 --- /dev/null +++ b/docs/vflow_logo_dark.svg @@ -0,0 +1 @@ +vflow diff --git a/docs/vflow_mark.svg b/docs/vflow_mark.svg new file mode 100644 index 0000000..58276d8 --- /dev/null +++ b/docs/vflow_mark.svg @@ -0,0 +1 @@ +vflow diff --git a/docs/vflow_mark_512.png b/docs/vflow_mark_512.png new file mode 100644 index 0000000000000000000000000000000000000000..a078c0b0dddd1719418b5729e9cb6e4a0de81dd6 GIT binary patch literal 14683 zcmd6ui$9b7|Hm&TBggwzNt)4tJ0wymXX)&i!<;hb!<`(8&Dn;cvr0}mC&yuq$ux&s zi*dHGz}u8%^BdKk+3b}*|Ry}`ILe3f6^;GT8xD`R!v8TV_b z=knh=_VlY5CGIPgz4mqQ-sjWq!(VFd-OSZb=P-;3hiWn%UNEk%gz3s;w*~ywy%jOj z!d%sv`I&JJEb0I6kLKO*kH2qM{v52P(!b8^^7x1h`IHf{Ah13I_P5$u;cbS@LUH4v z-6kKSJ3Aiu44-{HE!uh1#JWj(Zh3rP#yNAO5(FU#O1e?6(+@_b2r`r8 zGcM0bZ6*rC#7r};{~ppbqrxCa^Lz#KX~Cs&-}6oTjkXd_DXocV3s^Byx1WxXF^!}X z5L9!tu1#-zex=%zLAI3IhL^gpfA#Kc3Zr|Bm|8FdA$)SMpDXiQw+hF}_&4X8%!*~h zsetSe^=)~~I0SiLZ8^5?eA$7~ZE^g|ZX zH&W9Yuub6nCE=>oFE`wmwAMw-=YyQDW%PRvm~8J24S9QYsQCCy$pF}>*x}$rr*);u zmz?{Wr$$QIj^wuxbnsdFfk=}j&j#_k9wSM!K1u03JW%dQMwPEZeaFov%_Ae^*N-me z(Q5f2=a|Hj#>2^Lbw#a}hZSdWDDoJD7oGWZ*(@-Ttr+?b*1W!$RuM+!fe_sb<&ER> zm+AA~xuN)G%*A~?VO)k9-%#K z6V>tG*pth}dPSwlP!ylPPi@ANo-$C0*_7qjA2$WA)Dzj|c+WWL-)hnL_Pt+bVAQoHv# z%&rjXvmxmGhuN`+r|Bi^6kSN`ec0PCmK>FXj|EreyiP&dxfwwo35hD>e(Xr24C86e zg%q}T`Su``Gj*}yvR=0v{xmQks!+I3E;-Jxu*<2u4GMTx&DSz?FXEIXyOjJULWOvV zABx^e@z;r!Edp{$_f9Bt3I7#A(aE#o7W*!==f&V$z-Ul|`> zF#N6IZiO$Upk%_pj`^>3KkbhlRvTx01VZ{E`DRy~Yy4U8U&jgB4pbq?JDlsU7|X^& z(Kgc}@0)8>+IrSw!AX-BJQEMR1WqP3yiQxO{+gd;Gq`1PnZV?Qe3gbBBMp~G_LwVg z1BV_QFC+|vp7exlb457{HlYoP(@hJi z9AxO82ze?L>&mS6i)a%XrNd`N z9*oDcnZqv49?>10931WRt78%`^H!SuY?JEXi$`n+8`NqRJs|VTUJ5tOZvQ>vG%Nq9 z@u1P;8Mo!i5!dDA%iYKUQTyd(&S_2qlJsEN|J`yW~=toAl{C#!1=gCn*C}t@xw_A1XS>o_~$$xtSSFl4xtV>#EjZm^;t=}ZZ7;u`-{~;} z<0Y-{W|K;ynUf3T6uX@ur{rRi$)-?C?A3kIOkPdHj`c+IjkF}YQA6Rn)(z@h*zU%Y3Sw?U;^v_r zzP#kvffOiu=)m~K*44$Il35}Fq~My-;AUyX>slw@>B<;wGm2mhjnj;`4x8}Tvm*Y_ z;Fa$(jL&@@*cB} zX|o~Dt;kSuqj-E!{%@70sk7^05W=5Z1nh21N5%wX|2P_5NtMtL_PBx0Pizl)0_$EBb6 zhu4vCF2Aq|x;=bz9-t~}(i{~->`=&+?wI1exCtp**6CO&sIg3Ryw{4f;uZ(#H=Fi9 z4VIXiqqyi2)OjAgW#y3)a#M;BGA3I%q_)*RA<2!JOW753oXU}@Y@*+F^`Xe=x~^D_ z8|#sDLFkqDk5{){=ixN*?8ql9IZ}v&6}1QDqVV=0a&F_q)BGK|1=Wdh`kRtPbB3Wy z)Hq4~yf|sHBq&s$tfJqh+ERZWIfjwurJvKNWc^FadV7HJ@avST4tr#S{B*D>@P}2z z;b7)f4&&EMZA+pErLEkPcd>A#SHv-iV-3zQ^5|om9zkq@B*t?Qq90adZT58uczDQsWnlo7d!u9(M&^Ynn^LBzT# z3alb_)MO+dqnts)N{;uv($!(bHUtMa+?U68VBtDy&#{w3j}#-F~Fa-=c6 zac{wG{_%G7;hgC-{TunB)4#HI?H{SlnQ2L+M8ZXnUDq0&4{Dosk?uQ5-XeuPz$foB zc}%`2)Bdm;TCRK>nJYXTFQ{pTsBt+C;`uOUo!VA3AmDw~(PrT3ZHXyHNvrKi=*Xw2 za=Mj>>^w z4y&dl6~DhizNFuk8uG&_;#P_#p0YQCN045FBEkdy5lW`ssT!KqEfpQENS^&%*bf%- z*w%IAG;K-5YuDgNOBZ%wkhD6L;fd+gJfw!ujOapsm0Ew|m(o z18{FaKUZxzF{*eHu#Su{6MV;#FFrOUQ6-^x04_kDh@7kZFo?qZjw|lgUpu`{f-L{a zprPV@Z6CpTc@hI8eZx~U16omj5!!iZNoU23wYY(j*fXUc!+G#m-#w^A7Q-}dd4&Ba z9@v%o!hS(j!LiW>@z$kuYt|vF5G^BHld*Y*`+_n*RP#6buZW2|eRGFtwLIsQ4j`NC z5pQ!#HwW=yzWCx-<8}1M6V~szRrYN`=t0P$m#W zM=Puc`bJExi9+JnOe1y>U@v?>wn-8O;Hjd8{!}NhB;#p!#RiHG?_HF#?wKQE!xi6O zFDDd%15_l!=K5QtlTVU2%ffW<=C5#D-zC85TUT;K{;hvhmkA!PGimdaG8`yVb!h zn7fi=URS5fwwEX=RGX-Vu;cjNw3LvgJ?hZ>cewA_t|tk{uM_`Ll{O{q9L7KdrOlgf z!VvggH6Dg8rag`nA|Y#^k)#4PsJU~uN^H>pB#>_#v7;H<9#=E^VbGAiw{{AcMK|h{ zrAl$EEoyeR)6}l4b`T8D$kkaQY9`An7mEi3isE+}`K8Ug=@RQ|wKewxDGOwr+$rS) zxH4UL!yAv@HeCZ&NIpB;X+v=toj+;V^O@EN%l#?;sYy<3r~(!7crMOciN2!sR6Tmr z2=k9dy^0kBA#haUPSZ;`@M4f`-=L8>W3P>rhyi^`I&uQPJm=ZC(+W;n=c$3RL9ZVm z0x5FR-|tMh?Zm@;m0p$aG-zg|U&j`7i;Eq+vJ?>x?6mgvqfZ<(2nKn760B*#ofOR+ z>Gw>|15%H^cl`r3uKOxiGn>%lGx~XUMjE#AM}kXYNotc$LlO&0O&6$cvKfEZwtj|G z%SA^jk_0)Gws+=-Xwy8eT7;oR>v)XY!x=UVKK&{9y*}3x6?EC6>%-7z_d=x<8mu9! zfwO`T3fGXyrz?ru17}d6xJbXfGd1M|46HZk!SJMM!3}eGa3+>Lk~y|=j+V@uyYEqg z$R8{F)OIO`BVIDF%lxdm)44Mt&QEoly((9Gtgu?M>qINY-vT-kVOssU(ytyS5a%8y zWEY_&^oxC*TOYJ!uF_<)C%`VDw39puLg`{A8BW^(bD`p}IgnH&8x5PIf%jl=_dGXr+`W!vGf}HQ(##u3hy`}sj zcC=_q)+y=xrM0!Lhd%@sXl~!IaYezBovb|}cHNXTe>u4v7frI(H+pdN6Ws=0ft}G~ zW7ZEqWwOI{Y#mciYAme+XR+zkY3OL=uVqa|=H%Nh!%6)-%A@Oxk3ZItZOHukN@>}R z)1juMldnZttOj=3pWtJLPu}BDv*3yme{|DQncb|7;Y6c6^1CF@rgi!sokWiQPAOjd zJVy2C`NxZXUy9kCblJ_!we9=c zgVD(|pV}5*#NYXLb6zHR`Ryu22R>|ChD`+#_0xPeC~-Q=`R6xrHw%@rCpGoV6#`F; zU|&c<8?@N^%4zve*vnIHoKNLQ@AnUgA{%A61i^))pc0c*ADM+~97s0uOx=tDEmZk- zD*o*sKLO?{&OTZg{`lKHe0z*XUB*G}7s2ECF?agzh7kp~qAWud9MMuc^;=sa5}Vl~ zbS2x0pUI)xanjzZ)l9Pz7%94iGxht&0@0kYA7~qD2KAQ^{UQQvt&EE!Yy+wT=mvos z`CG!Vj_38>CCKS+7_P3a)3SIW@tZBd6Om;1F2r_Y%eE$)vW+M~_X=Ur#vqFKs6norON4$1-J+%c=RpZ-3zPqi? z1K}@~92M1Q;5cyhvn8yTWYl2MWo&N<$}uR7VbnJXaP-7@>iAoux4~B7HLFo&zfWqh z9N`C{7$Y9fv*x%<5M*vvJM1Dq;jd}Cxu~=#_GPIvuphY>g8Cl}ZQ)jWCpm@H(9w0T zT@Zv3m$F+{mY5ea_J9U)dd=|uY$*upwx}Rv-`QN$5?az9SZYqA^@u=_XS-rpim}2D zQiHG50*8elRoIu3QFW7N2|e!6AP1krC`thLX`Cn+;9oG=go8H6>otCvSs>wHhY$Ju z!`L3W%bAM9`x6qtg^i=hXK?SWm*~Yk(6bSrs7B*RV+cqpD7j#a&VQWS8hq?u-4Sft z_1oYsV=GW%>{=;EhA;zF-~`^CkGusz6Z+rFb@~*;^a#P5W>ziX?(B7?Uj(tgfDJ?o z(oUQ@vcrkk=d9x|%m>L{$e4ro>_-3k*3RQRZW0}fwC2j5gFW+Ki2gyFi+Vz;-%n@g zSx3HypdjsrXW=&LU{O66fckp8;aS&gD;w{{^C`yb?;<7)_-0xN#b2$?2C)GziwRC6 zT?U6CBr4{9%Hi6pp?d0S-1~Z(E^x&&rMaVX6dt{g_62lcB@4O9XGax7^h6BanN2F= z>)KSx9~V1-NYGUse`nDK&Rpws$fE-@Iqp08X)Le1d|M{6$c(s8I8^9=vE<_Jda>fx z2zigVWa)mg;Ui6M&-3+T-d9_*^(R0!H3T~3iLdVKtCjBS5`LYy>~D^d9&3G{10^@U zKefQ2An&`84V+OHO*MPj4*#0*H%9C)EJ&_?H@k#hU*(LuUIijFR{$^hUSQ!@LK@gn z4<>ccR=k7PUtGDV)+Z9Zek0w*2N6c76isT>g~&lj z74iJP=8v?c@-E1HsoaL~x19LG(9S-I?(_qQ9WxoelJL=&i@O-YMd1)+=@PSw#e@v} z3k^;Q6SXf>&F)_y)oG8yqMVC6fr00SGAo@ ztI+!(zVuAWShs{O0coQHND<>xLF+A!19Ca3Oi7IYSH3@>0SIK6cC$*h&O5d1RCP#T z@Z#bfjIBly-GZ@CWZ$T_7xp{juIyl8)EQZ8$1^}pu%K!|2$gKT|5ScnG8F$Va!f|J z{}Wdbk>BCiAbzWtywPh;gP?1`JJS2*sO?p|(b=RnZ~SUlN#Q38Hg!cdo0^9AFJ=nl z1lnCZK@^?k7lX8V(CLv6Nj*h*6QMgT-x&Qo=!D&48aZ?Y{FiU1wF6!Qk4MD^wGy6% zH;@L3@*cEp#mT;(vr4!aekL$;x4z}ju0_~NfZw%HpMLVC%nONm{#hY%3A*p*P=0+) zJrDJkgfad+L863-q_xS>%_#K9W?IaRT?alGkkA$YDi6xreT1iw!kIK zSq)!-e?-wa^mCpa^DmonMQL%OJk&fQim_jx`Djm||MmDgi9Tthnbdm|*Ga&DuVb!g z)V!cLP$Kzz*`=?8O$5%@hg!EkDG7gSC$Q=kHD?bj3DAe$u~Juh%8I$)9lLuOWppji zW}xhvERmA{m=djA41{*fB2Nuc9ePZNwY7du zft|2nAT5)M5PsNJvG!Km+xuj#KbMS%<&XyntAzwZTRYV4r99f{<~_T5+HK&Kpk$}8 z;JxU1xkYQ5_;*pLUWwv^KX%75FZ(-qci)~NxiwKCD@f9J^J4Z5q>LHC;e{=59SDr-K0|je1-g*tlIxfr5k^}eC z-H^HUpSMv&^FN>F$0yOm_lZGbB?FM<^CHBMS^`JsH#9NR-v#AFra}i%@fDd~eU?cXLY}x#I_rCD7Zl0*i0yI$u zHm0Ec?u0x>z)0!b_AXF*dn1K*!~CuO?BlOw?!E3Y+V=S=r+l14rkw^CCB>~4mKfTG z^UE@>{@Lc2989FZoZ_%$ub;AJy#rUrpv~5;xaU8C z`YH`1z#}{4g}|E+T7jkPIY*_~fK#G6m~|ygu^g82{*^I#F}Dur8S^QyHhvA=uvgmm zF9Pa-EaJY;C-$FO!{m4utiHoz?;W6w$$WEeC|oq>Gv#Ky-LAzmq52vjj78zH$3+fL>{MaRcyN#w^q8GTnt_I$+^y-g zassoL=dU8?V>VBO{qt@GN0x& zrP;5FWKZ>nGTmA*U8ZMT>g{9(G0KG05g$pNFK^xoRjW5%M|?4nf-KlfZm_VMw}#=| z_P*DlO_WbT0r{lDqCDHGZBr1Rumk)&)dnsP%#^ zpkZi4pNI^kjmhQ-DnIiMIZ+<^ar78X_chO`JEk7!#CBilH0gdF{@xbwbU%w%DDoNT zCT4OLf9+lELDh+i{B5#*!UjZN#o2LW#0eX!8-(y%7(QHf-tV4vZ{c-M@2$@}=h>mO z@3}3m$2fkWwi-F3Z?ZuGf#m14B$ke2D$IOmvYFT2-9J|MA2*Z$z~e+oD~a8}q2-z0 z*--|ed186~$3z<_kt?)+ijh`(9DRtNkwm)0D|c?W-)6y93H2VRE4OCsg+$$3XwgNr zoeh?V4P5;kBMN!!r|uiN2;b+TB0xcpo}3w&^#k`#>idO@-aX zr>|V@pL9%kC{K_q2^WOExk%c4GX96*aBr0U367)xx}wI|*Ld`v-nlxj)$yX%@b5{b zY^*8ZnWPj*Rl>0q<~7_R!6!6Uh8bHJ_lbK%ytG@%DM$Cx-IoB?%Ze=z&9eccE2Yap z{f0~8G7V=pT_nb`1eY5MLM_LCP4HZJexBa7adYtomYMpJD@cnjrzNMaEM_JaA&SAL zPQ_&QAbrd1zj|O-h;QTG3&l=OBzWxQA4f+92LJMjO8@_8rB0d2JXD;{FoxMKqw@epUey8_@1Sv_ox{*A6~)ht}5$LZ z9zO?KVy~hEP(JTm2P{WGUUUq5ARGZqEngxA%;tFVAX z_P_yENP8c-->FFV44Bps7m{FGe)eOHX^KHM#R=d7#)m+>zDUjznUE(m-6rnWG5ymg z?FMH5Mh2!|6TUiL@~5A=zSf5@OE}76dIe88VbZqGLQwOmk0jP3uPE9$GI%Plu$bGP zb?Ylni+%PMP;HIfu7*K)sVXp|;D%ilGhO}EMEPH%cWl^^?A{5}?;H06(4ucl%;m{2)BopHL$ zOeibI>&Y7DlFUzaWkSt7nWoDN_5OXRbx1NjGivNE=P(V*_C43inkixmLi$Au;fZLD z39Yw$oZf^prdk4(OQ}kpnLP0zg4SHlgjD-L6TJ`UVUYEvB2I@SA;(!Z^Q z{JJm25|=w*q_1AkH5TzYx-7% zf2eP|GzlHtxP3Mn>t1HJeY4>uD^oKyy^h&4R!l$FyK*S4s+zNheRy4z4YS#jY{`$g zN0v^oKx*~UP8WKali)J4-QY^ojl0N(`V*=Ob5z~;z#ZQrtJ>~uB zpfL2<~G}&Apx;YUARr2ngmOUW!mO9NhIJiQy^ zJit=cFQ^)zD>XS%9de|APkj+C#}Hc2+^tQkj&3$MYovHWDk_lGaHL5G5r zeVJjr<%-eQ_*dYaJ=G`YuYG>=I-Lh*n=W&irPpVRF)3c{>j6{Qf}g_sCr2|B^*r+j zGKuta=KjwuHe3NKdV7tfXG0Vdzm{3W*NBS1}_1Ju%9PB5Tafye*xa4JS0mJO1Z*_MdW_uGb^9n;%mN1EjF zqIGfYNmE4CWZC@FiQ@aT)4b;uULA8P5Ot&4;a6?~Wq;^ZQ~4qv;bo*PvSv~l%@I0y z;glo0So40;N0Df6jQoU(Rs5~gt^*y74uCfOK5OK!V+0zjJ13_d`sbkTI4NkmA3&n2 zYpjOY0Xf2efB-@QFIfURy9W?3JMz9}poc86JjhzkHb6{S(!s-X;k0}%jKRmYZr5$**}PI8`cBX-ymy!oBJ>2WUL z^9Z}mHQCvRkJoUgV0W1hyOf#qzWlE|tCaW|D*&b#YfrWmmx;9j%y#qgc|4PAp5dfrI8kxBk=T$Pm zpQdXF-H}1eExvYS7jF{(bAtz8aB@Wn=wv0-KTd%Y(zXcaBAf~AfvrMgqsZ% z9k-rvDQT6_&}+?Gt*Gx;-o8dl%J!EU=Nrkql{?Be6SFWbUyj~ zIlG5osA25Ek3Ug7z~?O}|C??Fm~hwHFEVYNN8YW>4jT<_+yZs1;eUda4{r^h#GP0! zmOclj-GI4uD8EyawubwAm-OS3xM9HsFq!LcUvDMHq0@E2hZ_X%Isoz0v6F2e)A(aI zmNQdo7xgOa96tFkF4dE%aVc+HN16jlDy0tXa2dJB6L%tL;(IMf<&Dh!3e|Lj(B3$% z!-fN^l$QX>(hkx5gUX3*@p(>}gK7>cIM$mYYI4hhwiv6-@0mI1!xQG3Y+by?h7dPG zzTl!+ZEwRHC{jc;u!@dU$0bgCtZgL|Ionj|6SBDlL5w` zgdsu9ej|Ro1t-0q47Y!ksN#f($r16&?@FKPzZnn6>)Acwn2jKbxCTn}=ufI1<2O`2 zy_Vc_8$i;P3Xy;9(mEdjerxxLZnpFA`-1{y%?CQPTMu+RB^Z)itUK9H7k6MH6^naYrk%QA8jt@tny!ln z@k_Q}*~-|hz5Kc!l6Rb{g~ozSSiLG?(h4QdlpH>e-X#1E{gu%4FMF)X{@$oocXreL zwz}`xY%yZ!hFAiyNVoKA7e3aH$aG9IG5HbX(;|K zS5d){{UFS-zcjaHRRPPmxF}XF$})Uu-Mw%c)?x-4&L^OMORgU9%zmxAfs)9Oue&xh z-f7ri>cRiYzc_q%&|`1X`)df2i=z6Th1%-6ta5Zp0f8bagk@sof-fI7-;2NliGpH z@ZV`DeXZfV7w*&wdnND1^%23|wthL?DMb&?T=zA$Ee09=RJ|^E>Z<2Vc)E6op+$k@ zxE1lLdbiRqQ#FHz^N5W^C&eXtt-1cp{Zu!Guc#CoFnqY z6t7i`T=RUy0HIZ5Pg40UE8^w6T@6u$32VxvZkb|8LaK`_((d$V>3oL%#2#dBUDRj^}^41npk0Kvd2I z)o(BSL~1p5;W;fT&cxkhqt;VkFjtguY<|ZkU`j#!5SN>K#NqR4;wbJ=3!1zSlhT89 ziBZoJUDOogZK`}YcC@9kctl$Md;3EbP$@mo{3%v7H}3mn4ac4Hgu9$yk^s#3I!0Wm zhd1IPnIB_qbkI-o&iHjWcB+$EtMilfjf>ri9T59A3Uq+?!uQ9dHmV_gFsY~_%WEMWG=5hPqnF(2$e z@A*ITt-?G$=f&GibYbREXEz_dA#v87_mceyv39Bgw;l?MuwzM#-0K9yzGEII{Md%E zRSCch#{R$_2go@tV~v!MWkxXy2uH!D23p zO6v5~M=dF@zq)(X{qvo+`7ep`MT@21)>#X`yi`tRkncAzqtAGNsoAuE_cn-Lll@R* z9(~=I%dOe27|2fpd}h4IEsjus_pj-nq9}Q6_)g)I!$672*v{^g4QSn&v+1@WNgQk5Ju$P*z5`jcs_ss*#87OJ3weDQIStrF;{`-K8?j^Dx)Z6!(J7tM2*zxxi zFDhwm(1q!5-Ncv`y0J?#q(|2ZM8Bhy)cHm~d+%<7x_)TVj?|H2v%rDSZ6Qk(RWXwuLz$e%DJnsqQO! zA(Mc%fxMNxy!>(YryLuPyec2vzmXz3l(rI2VZC}dvA2G^_ZAW0ineZNjFeL(49{QD zR6iS1-4zwQG=WeTpErlI6Hmabk_q9b(k4i?VCcCXpa-Jk8oChe1$jG2Xozk2h9rTu zNaxit&Uplz$&X*jq%|7LTda*0igsX$4=(SPd_NyEmz-IO_6IZC*hSI|s8o@s8kT%O zU$O726kZ2Rm;FuS{D$|CY~b!-+MDm~rH8SgRyYR*J-x87u`S)=(rV1D5BNLqc_30{l^x&7;i8ly4ogY;vo=_l453w+kpw18W36&!XiTT z9_&Fi>a)VZB=&^?N-)xm;Tw|nt&W{2|9cgTWD#H@5Vt1Su9ms-=69q$GOT|}DWoe$ zijO`1Cm7*?!uC!4{n%iq724lR@lm&a#ix$aaMciM<#r%KAORBfx7+1!8Ppz5+79M} zRE1im9#OI<^^MyjOc6o-JgppTL&eHH+PD~`pAKvSC}vN$ABFX+ggmO35{G)th2q}V z-IHA_`thNbBO-(nyFlUIpeTq=lEtyaZ(WKZV)pl<%_`^M&?yrx%ceo|KTl3N*_$8G zc3=TY`CA{*?IukDE#iCMKWwuW4h5-7T=H3q9yG9CY8>IBaWWm8v(1Jl-Ljyk(fXk9 zG%Ob=V*iA!m1-UwAscUc{UX|9V#3OJpv`Lg9{~-|q^;xKwNOi^ns(hARq8hqhMIZo zE-sGHDsB}HbF?{hF6-|15rUcxb|qw5$9p!slqz74NZCtWCmsXj6iiDvP$q}^te4)x zi8d(WVX%v?AoX_|*6iw1{IxNB?}i_HJ=UveKpeu)g4A4%3JorS)h2b`<(!b*Lry75 z5e47{oc)TYt`+v3UD;~&PQhqd!!Xy2Cduo;)`f;b4^5<(g7g1u` z11cd+Hu1i{OP)2f9yp-wMAik0mGPU2NOPHYa>oc0z7K>$ky98l&~2Re-T~3bMtY~- z5&N|v;t!AY;Q?P9;_E4|dpOq9Z=gK5yr`y^g836qz^uuUm~^VKEE@&TL1~QR4Jk_o z5KhiN1ogRfiV zV!phZF05^3^2Ye=$gizFw}zcSalaFl3~cSY%Fo7P(PgRcqUy~zYMs|RL;r{{FAjZd zse6@KgN!lHzQ)qFSkt9)DJ#jJtNH$Z#t0((jtu7v{OXtEyrDL3-5_WD0wWCurIS2e zcNf-a2F;2;v7oYJfYStLk+fbb-4d^sJ>x+*c6<+$`c zEt?Zw(Y5OrQO}vpCKp8A!fuSKaYk(x05CXAfTLw1M6O2yjng?f#&W|8w-|25blcS$ i!yTac|NC>Qiz7DGw0=vVMktB9%BvSmFO;8mj`=?)6SwRD literal 0 HcmV?d00001 diff --git a/docs/vfunc.html b/docs/vfunc.html index e11f4bc..54107c4 100644 --- a/docs/vfunc.html +++ b/docs/vfunc.html @@ -14,7 +14,7 @@ - + @@ -30,7 +30,7 @@

      vflow · Stability analysis for data-science pipelines, with PCS.

      diff --git a/docs/vset.html b/docs/vset.html index 42e78f5..d9174e7 100644 --- a/docs/vset.html +++ b/docs/vset.html @@ -14,7 +14,7 @@ - + @@ -30,7 +30,7 @@

      vflow · Stability analysis for data-science pipelines, with PCS.

      @@ -58,9 +58,13 @@ """ import hashlib +import inspect import os +import re +import types import warnings from copy import deepcopy +from functools import partial import joblib import numpy as np @@ -172,13 +176,14 @@ assert len(vfuncs) == len( vfunc_keys ), "vfuncs list and vfunc_keys list do not have the same length" - if len(set(map(repr, vfunc_keys))) < len(vfunc_keys): + # TODO: how best to handle tuple subkeys? + vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] + # compare as the dict below will (1 and 1.0 are one key) + if len(dict.fromkeys(vfunc_keys)) < len(vfunc_keys): raise ValueError( f"vfunc_keys must be unique; duplicates would silently " - f"drop vfuncs: {vfunc_keys}" + f"drop vfuncs: {[k[0].value for k in vfunc_keys]}" ) - # TODO: how best to handle tuple subkeys? - vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] else: vfunc_keys = [ (self.__create_subkey(f"{name}_{i}"),) for i in range(len(vfuncs)) @@ -227,9 +232,7 @@ # serve stale results, and it cannot hash lambdas at all. Key the # cache on the functions' code and the inputs instead. try: - cache_key = hashlib.sha256( - cloudpickle.dumps((out_dict, self._async, self._lazy, args)) - ).hexdigest() + cache_key = _cache_key(out_dict, self._async, self._lazy, args) except Exception as e: # noqa: BLE001 - fall back to no caching warnings.warn( f"Vset {self.name!r}: could not fingerprint the inputs for " @@ -253,36 +256,40 @@ if self._mlflow is not None: run_dict = {} - # log subkeys as params and value as metric - for k, v in out_dict.items(): - # only numeric scalars are metrics; fitted models, predictions - # and lazy promises are not logged - if k == PREV_KEY or not _is_numeric_scalar(v): - continue - origins = np.array([subk.origin for subk in k]) - # ignore init origins and the last origin (this Vset) - param_idx = [i for i in range(len(k[:-1])) if origins[i] != "init"] - # get or create mlflow run - run_dict_key = tuple(subk.value for subk in k[:-1]) - if run_dict_key in run_dict: - run_id = run_dict[run_dict_key] - else: - run = self._mlflow.create_run(self._exp_id) - run_id = run.info.run_id - run_dict[run_dict_key] = run_id - # log params - for idx in param_idx: - subkey = k[idx] - param_name = subkey.origin - # check if the origin occurs multiple times - if np.sum(origins == param_name) > 1: - occurence = np.sum(origins[:idx] == param_name) - param_name = param_name + str(occurence) - self._mlflow.log_param(run_id, param_name, subkey.value) - self._mlflow.log_metric(run_id, k[-1].value, float(v)) - # close the runs, or they stay RUNNING forever - for run_id in run_dict.values(): - self._mlflow.set_terminated(run_id) + try: + # log subkeys as params and value as metric + for k, v in out_dict.items(): + # only numeric scalars are metrics; fitted models, predictions + # and lazy promises are not logged + if k == PREV_KEY or not _is_numeric_scalar(v): + continue + origins = np.array([subk.origin for subk in k]) + # ignore init origins and the last origin (this Vset) + param_idx = [i for i in range(len(k[:-1])) if origins[i] != "init"] + # get or create mlflow run + # Subkeys compare by value and origin, so equal values + # from different Vsets get their own runs + run_dict_key = k[:-1] + if run_dict_key in run_dict: + run_id = run_dict[run_dict_key] + else: + run = self._mlflow.create_run(self._exp_id) + run_id = run.info.run_id + run_dict[run_dict_key] = run_id + # log params + for idx in param_idx: + subkey = k[idx] + param_name = subkey.origin + # check if the origin occurs multiple times + if np.sum(origins == param_name) > 1: + occurence = np.sum(origins[:idx] == param_name) + param_name = param_name + str(occurence) + self._mlflow.log_param(run_id, param_name, subkey.value) + self._mlflow.log_metric(run_id, _metric_name(k[-1].value), float(v)) + finally: + # close the runs, even if logging fails, or they stay RUNNING + for run_id in run_dict.values(): + self._mlflow.set_terminated(run_id) return out_dict def fit(self, *args): @@ -409,6 +416,15 @@ return Subkey(value, self.name, self._output_matching) +def _metric_name(value) -> str: + """An MLflow metric name for a subkey value: build_vset keys are tuples + like ('func=score', 'scale=1'), and MLflow allows only letters, digits, + and _ - . / : and spaces.""" + if isinstance(value, tuple): + value = "_".join(map(str, value)) + return re.sub(r"[^\w\-./: ]", "_", str(value)) + + def _is_numeric_scalar(v) -> bool: """Whether `v` can be logged as an MLflow metric.""" return isinstance(v, (int, float, np.integer, np.floating)) and not isinstance( @@ -416,6 +432,55 @@ ) +def _cache_key(out_dict: dict, is_async: bool, lazy: bool, args: tuple) -> str: + """A fingerprint of the vfuncs in `out_dict` and the data in `args`. + + The data is hashed with joblib, which hashes by value: a cloudpickle of it + would change with object identity (an output loaded back from the cache + is an equal copy that shares objects differently), so downstream Vsets + would miss the cache. The lineage entry (PREV_KEY) holds Vset objects, not + data, and is left out. The vfuncs are pickled with cloudpickle, which + pickles lambdas and functions defined in __main__ by value; functions from + importable modules are pickled by reference, so their source is added. + """ + data = tuple( + {k: v for k, v in a.items() if k != PREV_KEY} if isinstance(a, dict) else a + for a in args + ) + try: + data_hash = joblib.hash(data) + except Exception: # noqa: BLE001 - e.g. lazy promises holding lambdas + data_hash = hashlib.sha256(cloudpickle.dumps(data)).hexdigest() + h = hashlib.sha256(cloudpickle.dumps((out_dict, is_async, lazy))) + h.update(data_hash.encode()) + for source in _function_sources(out_dict.values()): + h.update(source.encode()) + return h.hexdigest() + + +def _function_sources(objs) -> list: + """Source code of the Python functions reachable from `objs` (unwrapping + Vfuncs, bound methods and partials), in a deterministic order.""" + sources, seen, stack = [], set(), list(objs)[::-1] + while stack: + obj = stack.pop() + if id(obj) in seen: + continue + seen.add(id(obj)) + if isinstance(obj, (Vfunc, AsyncVfunc)): + stack.append(obj.vfunc) + elif isinstance(obj, types.MethodType): + stack += [obj.__self__, obj.__func__] + elif isinstance(obj, partial): + stack += [*obj.keywords.values(), *obj.args, obj.func] + elif isinstance(obj, types.FunctionType): + try: + sources.append(inspect.getsource(obj)) + except (OSError, TypeError): # no source file, e.g. in a REPL + pass + return sources + + def _apply_func_cached( cache_key, out_dict: dict, is_async: bool, lazy: bool, args: tuple ): @@ -672,13 +737,14 @@

      Classes

      assert len(vfuncs) == len( vfunc_keys ), "vfuncs list and vfunc_keys list do not have the same length" - if len(set(map(repr, vfunc_keys))) < len(vfunc_keys): + # TODO: how best to handle tuple subkeys? + vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] + # compare as the dict below will (1 and 1.0 are one key) + if len(dict.fromkeys(vfunc_keys)) < len(vfunc_keys): raise ValueError( f"vfunc_keys must be unique; duplicates would silently " - f"drop vfuncs: {vfunc_keys}" + f"drop vfuncs: {[k[0].value for k in vfunc_keys]}" ) - # TODO: how best to handle tuple subkeys? - vfunc_keys = [(self.__create_subkey(k),) for k in vfunc_keys] else: vfunc_keys = [ (self.__create_subkey(f"{name}_{i}"),) for i in range(len(vfuncs)) @@ -727,9 +793,7 @@

      Classes

      # serve stale results, and it cannot hash lambdas at all. Key the # cache on the functions' code and the inputs instead. try: - cache_key = hashlib.sha256( - cloudpickle.dumps((out_dict, self._async, self._lazy, args)) - ).hexdigest() + cache_key = _cache_key(out_dict, self._async, self._lazy, args) except Exception as e: # noqa: BLE001 - fall back to no caching warnings.warn( f"Vset {self.name!r}: could not fingerprint the inputs for " @@ -753,36 +817,40 @@

      Classes

      if self._mlflow is not None: run_dict = {} - # log subkeys as params and value as metric - for k, v in out_dict.items(): - # only numeric scalars are metrics; fitted models, predictions - # and lazy promises are not logged - if k == PREV_KEY or not _is_numeric_scalar(v): - continue - origins = np.array([subk.origin for subk in k]) - # ignore init origins and the last origin (this Vset) - param_idx = [i for i in range(len(k[:-1])) if origins[i] != "init"] - # get or create mlflow run - run_dict_key = tuple(subk.value for subk in k[:-1]) - if run_dict_key in run_dict: - run_id = run_dict[run_dict_key] - else: - run = self._mlflow.create_run(self._exp_id) - run_id = run.info.run_id - run_dict[run_dict_key] = run_id - # log params - for idx in param_idx: - subkey = k[idx] - param_name = subkey.origin - # check if the origin occurs multiple times - if np.sum(origins == param_name) > 1: - occurence = np.sum(origins[:idx] == param_name) - param_name = param_name + str(occurence) - self._mlflow.log_param(run_id, param_name, subkey.value) - self._mlflow.log_metric(run_id, k[-1].value, float(v)) - # close the runs, or they stay RUNNING forever - for run_id in run_dict.values(): - self._mlflow.set_terminated(run_id) + try: + # log subkeys as params and value as metric + for k, v in out_dict.items(): + # only numeric scalars are metrics; fitted models, predictions + # and lazy promises are not logged + if k == PREV_KEY or not _is_numeric_scalar(v): + continue + origins = np.array([subk.origin for subk in k]) + # ignore init origins and the last origin (this Vset) + param_idx = [i for i in range(len(k[:-1])) if origins[i] != "init"] + # get or create mlflow run + # Subkeys compare by value and origin, so equal values + # from different Vsets get their own runs + run_dict_key = k[:-1] + if run_dict_key in run_dict: + run_id = run_dict[run_dict_key] + else: + run = self._mlflow.create_run(self._exp_id) + run_id = run.info.run_id + run_dict[run_dict_key] = run_id + # log params + for idx in param_idx: + subkey = k[idx] + param_name = subkey.origin + # check if the origin occurs multiple times + if np.sum(origins == param_name) > 1: + occurence = np.sum(origins[:idx] == param_name) + param_name = param_name + str(occurence) + self._mlflow.log_param(run_id, param_name, subkey.value) + self._mlflow.log_metric(run_id, _metric_name(k[-1].value), float(v)) + finally: + # close the runs, even if logging fails, or they stay RUNNING + for run_id in run_dict.values(): + self._mlflow.set_terminated(run_id) return out_dict def fit(self, *args): From a33800529d74d9050aca47e59072c99a700f3440 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Thu, 15 Oct 2026 17:16:14 -0400 Subject: [PATCH 6/6] Add docs/favicon.png (the **Icon* ignore rule caught it) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Xy8Dg7fAqocHGzMqVCiSK --- .gitignore | 1 + docs/favicon.png | Bin 0 -> 1655 bytes 2 files changed, 1 insertion(+) create mode 100644 docs/favicon.png diff --git a/.gitignore b/.gitignore index e554f41..08dd956 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ env **.idea *.pth **Icon* +!docs/favicon.png venv .vscode/** # Compiled python modules. diff --git a/docs/favicon.png b/docs/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..301add88a6e4f8cda63882fd5bb6d1a7e51d366d GIT binary patch literal 1655 zcmV--28j8IP)^@RCt{2T5C)kRTM^|tu$I_8a1>?kdngAV^<|wHCo%KB#oLj z(NtS))&B6WM*U&bxQpzPw$>JHiY~Lu(3Tk#ACcXaSDO$cvmhcgTC9dfttK-Nb^*&n z1Y|t3i#%uUF1vGg__1Gd0wLUc&iCDO&z<{7Nl7A!B$7xXiG+f*$!jQk|9xhMc+lby zpEVx|W*Hq*LQqh8bt8~2HQU9Y#UaUaW|yXDyZDyT@nJ)uQ@##JzX3M7i1-qt9n$kg z$ErIlCH+N^=DjH5^O+!CG&){gLEEL`sE9AHrG1d6dTt=y?f2)qTW>UiLtMzS2G>K{ zyUY%0CK~ck9|Ftz_@H+!%?MMJ%Rd9wMKZ#BluLM&tnFE21gFKY$*C6WUt1}A@K)5n znM?b&L3(#ZAwZd0dz$foPrLl`LIW{^@be^9*@5(p4Q6ZLpxHVgT5Lf%Y=Lh2o#8^w z2C{sOr9?ac8S%3!dArn>e`G*4&4KJkO9hxSMi`@*wx_TWRRe7sd~u%myn!gAv@KA8 zYz$dLJH%beXv6YX3kCA5#nE>N{(h}i6UyDTS5ts+Hq3w!yLBh328%tg* z{Z*6zkR89{@U2w-NCM=r-5EH1&%t6179q)pGvGs9{(p2bfe=+7?kgN zWm{`>9FdTFt~TA|;omem4#Plt_)~&G7?$_*)j;f#CNgc|*Kt~F3p}QTydwN z+Z=~Ds96K-;&u#NV!|e`cO9$(KWK>XddwX|n!l?qfY_^?^6JPTZYX;I5WxwAheIRp z;d)^7<15rA$3P~02aGKfphP^NQ{1Z`%dWTVV%KULeXPhjE~mj%_;pr*>D4s%69&3e zdQiFFgSG#Cqhm1)$^!=FP{imsB0<>#D+ST$I1+}O7B<6F&pu5wvSmaA`y<)NP8Kw< zJB*GCJ4{XN9jDMVFgh4QtV%p=|dq;u% zX7?tdvFhu^q~{H?pbK{lDYQ%2PBFrS$_2$ATTlBplbQ<}EE`1unlpkVlbM45r2-xcrBMaJFVgx}>DsLJ=0<4OK%(0UFrBm<~Ty0?H|D zi6j8jgL)jkVSbNCNdJ>K{4Rz~PKCKHp#|_y=2f@r{>=aaG$TfTOev6h`&0$(ZhIzD z7Lab?39a^7wY6!4mwy#@Y(CP%dBKho-kOIZ?J-P^m3P7@|E)6eGf*7&>(3MclJV~g zErFWvc7$@ZcFN%dQO8}>*t=gMJT?1KLIByjfpq7BW$zcViG3U&Qv~vvZ*L2|;9~s4 zDof&bYHeWDS>*X$XfMyV7x*~jeLwBvHssY;;(ZbmEpphF>!}L=V<5x{Am06d9a#35 z5h^s08;>Nw?19F=$^l1yUES)sgNO#X&dzk@V@Y}Dy72FL_0{X6A-~{Ee+C45+bAAb z00r;QC^k)!}|ZEsjm7S?4PhnB#}fCNhGlh{{aT7m=B)Od1L?p002ovPDHLkV1l0@ B2?hWF literal 0 HcmV?d00001