diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index bb67290..0d2a855 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,6 +1,6 @@ name: Check black coding style -on: [push, pull_request] +on: [push] jobs: lint: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index aceb841..f8d2669 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -23,7 +23,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install .[dev] + python -m pip install -e ".[dev]" - name: Test with pytest run: | pytest diff --git a/.gitignore b/.gitignore index 68bc17f..196050e 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Vscode +.vscode/ diff --git a/CI/unit_tests/measurements/test_base_measurement.py b/CI/unit_tests/measurements/test_base_measurement.py new file mode 100644 index 0000000..8dfd64f --- /dev/null +++ b/CI/unit_tests/measurements/test_base_measurement.py @@ -0,0 +1,118 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +Test the base measurement class. +""" + +from typing import Optional + +import numpy as np +import pytest +from numpy.testing import assert_raises + +from papyrus.measurements import BaseMeasurement + + +class DummyMeasurement(BaseMeasurement): + """ + Dummy measurement class for testing. + """ + + def __init__(self, name: str, rank: int): + super().__init__(name, rank) + + def apply( + self, a: np.ndarray, b: np.ndarray, c: Optional[np.ndarray] = None + ) -> np.ndarray: + if c is not None: + return a + b + c + return a + b + + +class TestBaseMeasurement: + """ + Test the base measurement class. + """ + + def test_init(self): + """ + Test the constructor method of the BaseMeasurement class. + """ + # Test the constructor method + name = "test" + rank = 1 + measurement = BaseMeasurement(name, rank) + assert measurement.name == name + assert measurement.rank == rank + + # Test the rank parameter + with pytest.raises(ValueError): + BaseMeasurement(name, -1) + + def test_call(self): + """ + Test the call method of the BaseMeasurement class. + """ + # Test the call method + name = "test" + rank = 1 + measurement = BaseMeasurement(name, rank) + + # Test the apply method + with pytest.raises(NotImplementedError): + measurement.apply() + + # Test the apply method + a = np.array([[1, 2], [3, 4]]) + b = np.array([[5, 6], [7, 8]]) + c = np.array([[9, 10], [11, 12]]) + + # Test the call method with only arguments + measurement = DummyMeasurement(name, rank) + result = measurement(a, b) + assert np.allclose(result, a + b) + result = measurement(a, b, c) + assert np.allclose(result, a + b + c) + + # Test the call method with only keyword arguments + result = measurement(a=a, b=b) + assert np.allclose(result, a + b) + result = measurement(a=a, b=b, c=c) + assert np.allclose(result, a + b + c) + + # Test the call method with both arguments and keyword arguments + result = measurement(a, b, c=c) + assert np.allclose(result, a + b + c) + result = measurement(a, b=b) + assert np.allclose(result, a + b) + + # Test error handling for wrong size of arguments + a = np.array([1, 2, 3]) + b = np.array([[4, 5, 6], [7, 8, 9]]) + c = np.array([[10, 11, 12], [13, 14, 15]]) + with assert_raises(ValueError): + measurement(a, b, c) + with assert_raises(ValueError): + measurement(a=a, b=b, c=c) + with assert_raises(ValueError): + measurement(a, b=b, c=c) + with assert_raises(ValueError): + measurement(a, b, c=c) diff --git a/CI/unit_tests/neural_state/test_neural_state.py b/CI/unit_tests/neural_state/test_neural_state.py new file mode 100644 index 0000000..d3ebf3a --- /dev/null +++ b/CI/unit_tests/neural_state/test_neural_state.py @@ -0,0 +1,69 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +from papyrus.neural_state import NeuralState + + +class TestNeuralState: + + def test_init(self): + + neural_state = NeuralState() + assert neural_state.loss is None + assert neural_state.accuracy is None + assert neural_state.predictions is None + assert neural_state.targets is None + assert neural_state.ntk is None + + neural_state = NeuralState( + loss=[], + accuracy=[], + predictions=[], + targets=[], + ntk=[], + ) + assert neural_state.loss == [] + assert neural_state.accuracy == [] + assert neural_state.predictions == [] + assert neural_state.targets == [] + assert neural_state.ntk == [] + + def test_get_dict(self): + + neural_state = NeuralState() + assert neural_state.get_dict() == {} + + neural_state = NeuralState( + loss=[], + accuracy=[], + predictions=[], + targets=[], + ntk=[], + ) + assert neural_state.get_dict() == { + "loss": [], + "accuracy": [], + "predictions": [], + "targets": [], + "ntk": [], + } diff --git a/CI/unit_tests/neural_state/test_neural_state_creator.py b/CI/unit_tests/neural_state/test_neural_state_creator.py new file mode 100644 index 0000000..4c30faf --- /dev/null +++ b/CI/unit_tests/neural_state/test_neural_state_creator.py @@ -0,0 +1,65 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +import numpy as np + +from papyrus.neural_state import NeuralStateCreator + + +class TestNeuralStateCreator: + def test_init(self): + def network_apply_fn(params: dict, data: dict): + return np.arange(10) + + def ntk_apply_fn(params: dict, data: dict): + return np.arange(10) + + neural_state_creator = NeuralStateCreator( + network_apply_fn=network_apply_fn, + ntk_apply_fn=ntk_apply_fn, + ) + assert neural_state_creator.apply_fns == { + "predictions": network_apply_fn, + "ntk": ntk_apply_fn, + } + + def test_apply(self): + def network_apply_fn(params: dict, data: dict): + return np.arange(10) + + def ntk_apply_fn(params: dict, data: dict): + return np.arange(10) + + neural_state_creator = NeuralStateCreator( + network_apply_fn=network_apply_fn, + ntk_apply_fn=ntk_apply_fn, + ) + + neural_state = neural_state_creator( + params={}, + data={}, + loss=np.arange(5), + ) + assert np.all(neural_state.predictions == np.arange(10)) + assert np.all(neural_state.ntk == np.arange(10)) + assert np.all(neural_state.loss == np.arange(5)) diff --git a/CI/unit_tests/test_dummy_test.py b/CI/unit_tests/test_dummy_test.py deleted file mode 100644 index 57b89c2..0000000 --- a/CI/unit_tests/test_dummy_test.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Some Information about this module -""" - - -class TestDummyTest: - """ - Some Information about this class - """ - - def test_dummy_test(self): - """ - Some Information about this method - """ - assert True diff --git a/CI/unit_tests/utils/test_analysis_utils.py b/CI/unit_tests/utils/test_analysis_utils.py new file mode 100644 index 0000000..92bfbed --- /dev/null +++ b/CI/unit_tests/utils/test_analysis_utils.py @@ -0,0 +1,80 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +import numpy as np +from numpy.testing import assert_almost_equal + +from papyrus.utils import ( + compute_shannon_entropy, + compute_trace, + compute_von_neumann_entropy, +) + + +class TestAnalysisUtils: + """ + Test suite for the analysis utils. + """ + + def test_compute_trace(self): + """ + Test the computation of the trace. + """ + vector = np.random.rand(10) + matrix = np.diag(vector) + + # Test the trace without normalization + trace = compute_trace(matrix, normalize=False) + assert trace == np.sum(vector) + + # Test the trace with normalization + trace = compute_trace(matrix, normalize=True) + assert trace == np.sum(vector) / 10 + + def test_shannon_entropy(self): + """ + Test the Shannon entropy. + """ + dist = np.array([0.2, 0.2, 0.2, 0.2, 0.2]) + assert_almost_equal(compute_shannon_entropy(dist), np.log(5)) + assert_almost_equal(compute_shannon_entropy(dist, effective=True), 1.0) + + dist = np.array([0, 0, 0, 0, 1]) + assert compute_shannon_entropy(dist) == 0 + assert compute_shannon_entropy(dist, effective=True) == 0 + + dist = np.array([0, 0, 0, 0.5, 0.5]) + assert compute_shannon_entropy(dist) == np.log(2) + s = compute_shannon_entropy(dist, effective=True) + assert s == np.log(2) / np.log(5) + + def test_compute_von_neumann_entropy(self): + """ + Test the computation of the von-Neumann entropy. + """ + matrix = np.eye(2) * 0.5 + entropy = compute_von_neumann_entropy(matrix=matrix, effective=False) + assert entropy == np.log(2) + + entropy = compute_von_neumann_entropy(matrix=matrix, effective=True) + assert entropy == 1 diff --git a/CI/unit_tests/utils/test_matrix_utils.py b/CI/unit_tests/utils/test_matrix_utils.py new file mode 100644 index 0000000..4a1bc2b --- /dev/null +++ b/CI/unit_tests/utils/test_matrix_utils.py @@ -0,0 +1,226 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +import numpy as np +from numpy.testing import assert_array_almost_equal, assert_array_equal, assert_raises + +from papyrus.utils import ( + compute_gramian_diagonal_distribution, + compute_hermitian_eigensystem, + compute_l_pq_norm, + flatten_rank_4_tensor, + normalize_gram_matrix, + unflatten_rank_4_tensor, +) + + +class TestMatrixUtils: + """ + Test the matrix utils class. + """ + + def test_compute_hermitian_eigensystem(self): + """ + Test the computation of the eigensystem of a hermitian matrix. + """ + # Create a dummy hermitian matrix + # eigenvalues + l1 = 1 + l2 = 2 + l3 = 3 + # eigenvectors + v1 = np.array([1, 1, 0]) / np.sqrt(2) + v2 = np.array([1, -1, 0]) / np.sqrt(2) + v3 = np.array([0, 0, 1]) + # create the matrix + M = l1 * np.outer(v1, v1.conj()) + M += l2 * np.outer(v2, v2.conj()) + M += l3 * np.outer(v3, v3.conj()) + + # check if it is hermitian + assert np.allclose(M, M.T.conj()) + + # assert that the eigenvalues and eigenvectors are correct + + # normalize = False + vals, vects = compute_hermitian_eigensystem(M, normalize=False) + assert_array_almost_equal(vals, [3, 2, 1]) + assert_array_almost_equal(vects, np.array([v3, -v2, -v1]).T) + + # normalize = True + vals, vects = compute_hermitian_eigensystem(M, normalize=True) + assert_array_almost_equal(vals, [3 / 6, 2 / 6, 1 / 6]) + assert_array_almost_equal(vects, np.array([v3, -v2, -v1]).T) + + def test_normalizing_gram_matrix(self): + """ + Test the normlize gram matrix function. + + We fix the diagonals and test whether it performs the correct operation. + Note that this is not a correctly normalized covariance matrix rather + one that can be tested well to properly scale under the normalization procedure. + """ + # 4x4 covariance matrix + matrix = np.random.uniform(low=0, high=3, size=(4, 4)) + + # Fix diagonals + for i in range(4): + matrix[i, i] = i + 3 + + normalized_matrix = normalize_gram_matrix(matrix) + + # Assert diagonals are 1 + assert_array_almost_equal( + np.diagonal(normalized_matrix), np.array([1.0, 1.0, 1.0, 1.0]) + ) + + # Test 1st row + row = 0 + row_mul = row + 3 + multiplier = np.sqrt( + np.array([3 * row_mul, 4 * row_mul, 5 * row_mul, 6 * row_mul]) + ) + truth_array = matrix[row] / multiplier + assert_array_almost_equal(normalized_matrix[row], truth_array) + + # Test 2nd row + row = 1 + row_mul = row + 3 + multiplier = np.sqrt( + np.array([3 * row_mul, 4 * row_mul, 5 * row_mul, 6 * row_mul]) + ) + truth_array = matrix[row] / multiplier + assert_array_almost_equal(normalized_matrix[row], truth_array) + + # Test 3rd row + row = 2 + row_mul = row + 3 + multiplier = np.sqrt( + np.array([3 * row_mul, 4 * row_mul, 5 * row_mul, 6 * row_mul]) + ) + truth_array = matrix[row] / multiplier + assert_array_almost_equal(normalized_matrix[row], truth_array) + + # Test 4th row + row = 3 + row_mul = row + 3 + multiplier = np.sqrt( + np.array([3 * row_mul, 4 * row_mul, 5 * row_mul, 6 * row_mul]) + ) + truth_array = matrix[row] / multiplier + assert_array_almost_equal(normalized_matrix[row], truth_array) + + # Test encoutering zeros + matrix[0, 0] = 0 + normalized_matrix = normalize_gram_matrix(matrix) + assert_array_almost_equal( + np.diagonal(normalized_matrix), np.array([0.0, 1.0, 1.0, 1.0]) + ) + # Test 1st row + assert_array_almost_equal(normalized_matrix[0], np.zeros(4)) + assert_array_almost_equal(normalized_matrix[:, 0], np.zeros(4)) + + def test_compute_gramian_diagonal_distribution(self): + """ + Test the computation of the gramian diagonal distribution. + + The test is done in the following steps: + - Compute a gram matrix + - Compute magnitude density + - Compare to norm of vectors + """ + # Create a random array + array = np.random.random((7, 10)) + # Compute a scalar product matrix of the array + matrix = np.einsum("ij, kj -> ik", array, array) + # Compute the density of array amplitudes + array_norm = np.linalg.norm(array, ord=2, axis=1) + array_norm_density = array_norm / array_norm.sum() + + # Evaluate the magnitude density with the function that is to be tested + mag_density = compute_gramian_diagonal_distribution(matrix) + assert_array_almost_equal(array_norm_density, mag_density) + + def test_compute_l_pq_norm(self): + """ + Test the computation of the L_pq norm of a matrix. + + The test is done in the following steps: + - Create a random matrix + - Compute the L_pq norm + - Compare to the numpy implementation + """ + # Create a random matrix + matrix = np.random.random((5, 5)) + # Compute the L_pq norm + p = 2 + q = 2 + norm = compute_l_pq_norm(matrix, p, q) + # Compute the numpy implementation + norm_numpy = np.linalg.norm(matrix, ord="fro") + assert_array_almost_equal(norm, norm_numpy) + + def test_flatten_rank_4_tensor(self): + """ + Test the flattening of a rank 4 tensor. + """ + # Check for assertion error + tensor = np.arange(24).reshape((2, 3, 4)) + assert_raises(ValueError, flatten_rank_4_tensor, tensor) + + # Check the flattening for specific tensor + tensor = np.arange(4 * 4).reshape(2, 2, 2, 2) + assertion_matrix = np.array( + [[0, 1, 4, 5], [2, 3, 6, 7], [8, 9, 12, 13], [10, 11, 14, 15]] + ) + flattened_tensor, shape = flatten_rank_4_tensor(tensor) + assert_array_equal(flattened_tensor, assertion_matrix) + assert shape == (2, 2, 2, 2) + + # Check flattened shape for random tensor + tensor = np.random.random((3, 4, 5, 6)) + flattened_tensor, _ = flatten_rank_4_tensor(tensor) + assert flattened_tensor.shape == (15, 24) + + def test_unflatten_rank_4_tensor(self): + """ + Test the unflattening of a rank 2 tensor. + + It should invert the operation of flatten_rank_4_tensor. + """ + # Check for assertion errors + tensor = np.arange(24).reshape((6, 4)) + # Error for wrong shape of new tensor + assert_raises(ValueError, unflatten_rank_4_tensor, tensor, (2, 2)) + # Error for dimension mismatch + assert_raises(ValueError, unflatten_rank_4_tensor, tensor, (2, 3, 2, 4)) + # Error for wrong shape of tensor + tensor = np.arange(24).reshape((2, 3, 4)) + assert_raises(ValueError, unflatten_rank_4_tensor, tensor, (2, 3, 2, 2)) + + # Check the unflattening + shape = (2, 3, 4, 5) + tensor = np.arange(2 * 3 * 4 * 5).reshape(*shape) + flattened_tensor, _ = flatten_rank_4_tensor(tensor) + unflattened_tensor = unflatten_rank_4_tensor(flattened_tensor, shape) + assert_array_equal(unflattened_tensor, tensor) diff --git a/examples/measurements.ipynb b/examples/measurements.ipynb new file mode 100644 index 0000000..83c2ed9 --- /dev/null +++ b/examples/measurements.ipynb @@ -0,0 +1,273 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to Measurements\n", + "\n", + "## Overview\n", + "This notebook shows how to use the `measurements` of the papyrus package. \n", + "It will start by showing how to create a `Measurement` object and how to connect it \n", + "to a recorder. " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from papyrus.measurements import Loss\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Neural state keys: ['loss', 'predictions', 'targets']\n", + "The loss is [[0.25]\n", + " [0.36]]\n" + ] + } + ], + "source": [ + "# Define a measurement for a loss function\n", + "\n", + "def loss_fn(predictions, targets):\n", + " return np.mean((predictions - targets) ** 2, keepdims=True)\n", + "\n", + "loss = Loss(\n", + " name='loss', # Name of the measurement\n", + " apply_fn=loss_fn # The function that will be called to compute the loss\n", + ")\n", + "print(f\"Neural state keys: {loss.neural_state_keys}\")\n", + "\n", + "# Defining the neural state\n", + "neural_state = {\n", + " 'predictions': np.array([[0.5], [0.6]]),\n", + " 'targets': np.array([[0.], [0.]])\n", + "}\n", + "\n", + "print(f\"The loss is {loss( **neural_state )}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, calling a `Measurement` object will execute the measurement and return the result." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Defining Neural States\n", + "\n", + "The concept behind neural states is to define the state of a neural network at a given time.\n", + "This could theoretically take any form and the recorders allow for the definition of custom states.\n", + "However, the default state can take the following form:\n", + "```python\n", + "neural_state = {\n", + " \"loss\": np.ndarray, \n", + " \"accuracy\": np.ndarray,\n", + " \"predictions\": np.ndarray,\n", + " \"targets\": np.ndarray,\n", + " \"ntk\": np.ndarray,\n", + "}\n", + "```\n", + "Note that this defines one neural state.\n", + "The keys of the dictionary have to match the keys of the `Measurement` objects. \n", + "\n", + "Each value of a neural state has to be a numpy array, with its first dimension defining \n", + "the number of sub-states, the entire neural state is composed of.\n", + "For all keys, the number of sub-states has to be the same." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Default Measurements" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from papyrus.measurements import *" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Loss and Accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The loss is [[0.25]\n", + " [0.36]]\n", + "The loss is [[0.5]\n", + " [0.6]]\n" + ] + } + ], + "source": [ + "#############################################\n", + "### Defining a loss function as a measurement\n", + "#############################################\n", + "\n", + "def loss_fn(predictions, targets):\n", + " return np.mean((predictions - targets) ** 2, keepdims=True)\n", + "\n", + "loss = Loss(\n", + " name='loss', # Name of the measurement\n", + " apply_fn=loss_fn # The function that will be called to compute the loss\n", + ")\n", + "\n", + "# Defining the neural state\n", + "neural_state = {\n", + " 'predictions': np.array([[0.5], [0.6]]),\n", + " 'targets': np.array([[0.], [0.]])\n", + "}\n", + "\n", + "print(f\"The loss is {loss( **neural_state )}\")\n", + "\n", + "\n", + "#############################################\n", + "### Measuring pre-computed loss values \n", + "#############################################\n", + "\n", + "loss = Loss(name='loss')\n", + "\n", + "# Defining the neural states with a pre-computed loss\n", + "neural_state = { 'loss': np.array([ [0.5], [0.6] ]) }\n", + "\n", + "print(f\"The loss is {loss(**neural_state)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The accuracy is [0.66666667]\n", + "The accuracy is [[0.8 0.6 0.3]\n", + " [0.2 0.4 0.7]]\n" + ] + } + ], + "source": [ + "#############################################\n", + "### Defining an accuracy measurement\n", + "#############################################\n", + "\n", + "def accuracy_fn(predictions, targets):\n", + " return np.sum(np.argmax(predictions, axis=1) == np.argmax(targets, axis=1)) / len(predictions)\n", + "\n", + "neural_state = {\n", + " 'predictions': np.array([ [[0.8, 0.2], [0.6, 0.4], [0.3, 0.7]] ]),\n", + " 'targets': np.array([ [[1, 0], [0, 1], [0, 1]] ])\n", + "}\n", + "\n", + "accuracy = Accuracy(\n", + " name='accuracy',\n", + " accuracy_fn=accuracy_fn\n", + ")\n", + "\n", + "print(f\"The accuracy is {accuracy(**neural_state)}\")\n", + "\n", + "\n", + "#############################################\n", + "### Measuring pre-computed accuracy values\n", + "#############################################\n", + "\n", + "accuracy = Accuracy(name='accuracy')\n", + "\n", + "neural_state = { 'accuracy': np.array([ [0.8, 0.6, 0.3] , [0.2, 0.4, 0.7] ])}\n", + "\n", + "print(f\"The accuracy is {accuracy(**neural_state)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### NTK Properties" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "odict_keys(['predictions', 'targets'])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import inspect\n", + "\n", + "a = inspect.signature(accuracy_fn).parameters\n", + "a.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "papyrus", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/papyrus/__init__.py b/papyrus/__init__.py index 5c85bbb..1c18282 100644 --- a/papyrus/__init__.py +++ b/papyrus/__init__.py @@ -1,8 +1,31 @@ """ -Papyrus -======= -Papyrus is a simple, lightweight Python library for recording neural learning. +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ Summary ------- +papyrus measurements api. """ + +from papyrus import measurements, neural_state, utils + +__all__ = [ + measurements.__name__, + utils.__name__, + neural_state.__name__, +] diff --git a/papyrus/measurements/__init__.py b/papyrus/measurements/__init__.py new file mode 100644 index 0000000..0c19e37 --- /dev/null +++ b/papyrus/measurements/__init__.py @@ -0,0 +1,46 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +from papyrus.measurements.base_measurement import BaseMeasurement +from papyrus.measurements.measurements import ( + NTK, + Accuracy, + Loss, + NTKEigenvalues, + NTKEntropy, + NTKMagnitudeDistribution, + NTKSelfEntropy, + NTKTrace, +) + +__all__ = [ + BaseMeasurement.__name__, + NTKTrace.__name__, + NTKEntropy.__name__, + NTKSelfEntropy.__name__, + NTKEigenvalues.__name__, + NTKMagnitudeDistribution.__name__, + Loss.__name__, + Accuracy.__name__, + NTK.__name__, +] diff --git a/papyrus/measurements/base_measurement.py b/papyrus/measurements/base_measurement.py new file mode 100644 index 0000000..efc5a73 --- /dev/null +++ b/papyrus/measurements/base_measurement.py @@ -0,0 +1,146 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +from abc import ABC +from inspect import signature +from typing import List + +import numpy as np + + +class BaseMeasurement(ABC): + """ + Base class for all measurements. + + A measurement is a class that records an aspect of the learning process. + + Note + ---- + All measurements should inherit from this class and implement the apply method. + + Measurements that return arrays with sizes that depend on the number of inputs + **cannot** be applied on varying number of inputs. This is because the number of + dimensions of the input need to be same for all subsequent calls, otherwise an error + will be raised when storing the results in the database. + + Attributes + ---------- + name : str + The name of the measurement, defining how the instance in the database will + be identified. + rank : int + The rank of the measurement, defining the tensor order of the measurement. + neural_state_keys : List[str] + The keys of the neural state that the measurement takes as input. + A neural state is a dictionary of numpy arrays that represent the state of + a neural network. The keys are the names of the kwargs of the apply method. + """ + + def __init__(self, name: str, rank: int): + """ + Constructor method of the BaseMeasurement class. + + name : str + The name of the measurement, defining how the instance in the database + will be identified. + rank : int + The rank of the measurement, defining the tensor order of the + measurement. + """ + self.name = name + self.rank = rank + + if not isinstance(self.rank, int) or self.rank < 0: + raise ValueError("Rank must be a positive integer.") + + # Get the neural state keys that the measurement takes as input + self.neural_state_keys: List[str] = [] + self.neural_state_keys.extend(signature(self.apply).parameters.keys()) + + def apply(self, *args: np.ndarray, **kwargs: np.ndarray) -> np.ndarray: + """ + Method to perform a measurement. + + This method should be implemented in the child class. + It can take any number of arguments and keyword arguments and should return + the result of the function applied to the measurement. + + Note that this method performs a single measurement, on the diven input. + + Parameters + ---------- + *args : np.ndarray + The arguments to the function. + **kwargs : np.ndarray + The keyword arguments to the function. + + + Returns + ------- + np.ndarray + The result of the function applied to the measurement. + """ + raise NotImplementedError("Implemented in child class.") + + def __call__(self, *args: np.ndarray, **kwargs: np.ndarray) -> np.ndarray: + """ + Method to perform the measurement. + + This method calls the apply_fn method on each input and returns the result. + + It can take any number of arguments and keyword arguments and should return + the measurement result. + All inputs should be numpy arrays, with their first dimension indicating the + number of eqivalent measurements to be performed. The remaining dimensions + should be the same for all inputs, as they are the arguments to the function. + In case of one input, the first dimension should be 1. + + Returns + ------- + np.ndarray + The result of the measurement. + """ + # Get the number of arguments + num_args = len(args) + + # Get the keys and values of the keyword arguments if any + keys = list(kwargs.keys()) + vals = list(kwargs.values()) + + # Assert whether the length of dimension 0 of all inputs is the same + try: + inputs = args + tuple(vals) + assert all([len(i) == len(inputs[0]) for i in inputs]) + except AssertionError: + raise ValueError( + f"The first dimension of all inputs to the {self.name} measurement " + "must be the same." + ) + + # Zip the arguments and values + z = zip(*args, *vals) + + # Perform the measurement on each set of inputs + return np.array( + [self.apply(*i[:num_args], **dict(zip(keys, i[num_args:]))) for i in z] + ) diff --git a/papyrus/measurements/measurements.py b/papyrus/measurements/measurements.py new file mode 100644 index 0000000..e5f98aa --- /dev/null +++ b/papyrus/measurements/measurements.py @@ -0,0 +1,675 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +Module containing default measurements for recording neural learning. +""" + +from typing import Callable, Optional + +import numpy as np + +from papyrus.measurements.base_measurement import BaseMeasurement +from papyrus.utils.analysis_utils import ( + compute_shannon_entropy, + compute_trace, + compute_von_neumann_entropy, +) +from papyrus.utils.matrix_utils import ( + compute_gramian_diagonal_distribution, + compute_hermitian_eigensystem, +) + + +class Loss(BaseMeasurement): + """ + Measurement class to record the loss of a neural network. + + Neural State Keys + ----------------- + predictions : np.ndarray + The predictions of the neural network. Required if the loss function is + provided. Needs to be combined with the targets key. + targets : np.ndarray + The target values of the neural network. Required if the loss function is + provided. Needs to be combined with the predictions key. + loss : float + The loss of the neural network. Required if the loss function is not + provided. Allows to measure precomputed loss values. + """ + + def __init__( + self, + name: str = "loss", + rank: int = 0, + apply_fn: Optional[Callable] = None, + ): + """ + Constructor method of the Loss class. + + Parameters + ---------- + name : str (default="loss") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=0) + The rank of the measurement, defining the tensor order of the + measurement. + apply_fn : Optional[Callable] (default=None) + The loss function to be used to compute the loss of the neural network. + If the loss function is not provided, the apply method will assume that + the loss is used as the input. + If the loss function is provided, the apply method will assume that the + neural network outputs and the target values are used as inputs. + """ + super().__init__(name, rank) + self.apply_fn = apply_fn + + def apply( + self, + loss: Optional[float] = None, + predictions: Optional[np.ndarray] = None, + targets: Optional[np.ndarray] = None, + ) -> float: + """ + Method to record the loss of a neural network. + + Parameters need to be provided as keyword arguments. + + Parameters + ---------- + loss : Optional[float] (default=None) + The loss of the neural network. + predictions : Optional[np.ndarray] (default=None) + The predictions of the neural network. + targets : Optional[np.ndarray] (default=None) + The target values of the neural network. + + Returns + ------- + loss : float + The loss of the neural network. + """ + # Check if any of the inputs are None + if loss is None and (predictions is None or targets is None): + raise ValueError( + "Either the loss or the predictions and targets must be provided." + ) + # Check if a loss value and the predictions and targets are provided + if loss is not None and (predictions is not None or targets is not None): + raise ValueError( + "Either the loss or the predictions and targets must be provided." + ) + # If the loss is provided, return the loss + if loss is not None: + return loss + # If the loss is not provided, compute the loss using the loss function + return self.apply_fn(predictions, targets) + + +class Accuracy(BaseMeasurement): + """ + Measurement class to record the accuracy of a neural network. + + Neural State Keys + ----------------- + predictions : np.ndarray + The predictions of the neural network. Required if the loss function is + provided. Needs to be combined with the targets key. + targets : np.ndarray + The target values of the neural network. Required if the loss function is + provided. Needs to be combined with the predictions key. + accuracy : float + The accuracy of the neural network. Required if the accuracy function is not + provided. Allows to measure precomputed loss values. + """ + + def __init__( + self, + name: str = "accuracy", + rank: int = 0, + public: bool = False, + apply_fn: Optional[Callable] = None, + ): + """ + Constructor method of the Accuracy class. + + Parameters + ---------- + name : str (default="accuracy") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=0) + The rank of the measurement, defining the tensor order of the + measurement. + apply_fn : Optional[Callable] (default=None) + The accuracy function to be used to compute the accuracy of the neural + network. + # If the accuracy function is not provided, the apply method will assume + that the accuracy is used as the input. + If the accuracy function is provided, the apply method will assume that + the neural network outputs and the target values are used as inputs. + """ + super().__init__(name, rank) + self.apply_fn = apply_fn + + def apply( + self, + accuracy: Optional[float] = None, + predictions: Optional[np.ndarray] = None, + targets: Optional[np.ndarray] = None, + ) -> float: + """ + Method to record the accuracy of a neural network. + + Parameters need to be provided as keyword arguments. + + Parameters + ---------- + accuracy : Optional[float] (default=None) + The accuracy of the neural network. + predictions : Optional[np.ndarray] (default=None) + The predictions of the neural network. + targets : Optional[np.ndarray] (default=None) + The target values of the neural network. + + Returns + ------- + accuracy : float + The accuracy of the neural network. + """ + # Check if any of the inputs are None + if accuracy is None and (predictions is None or targets is None): + raise ValueError( + "Either the accuracy or the predictions and targets must be provided." + ) + # Check if a loss value and the predictions and targets are provided + if accuracy is not None and (predictions is not None or targets is not None): + raise ValueError( + "Either the accuracy or the predictions and targets must be provided." + ) + # If the accuracy is provided, return the accuracy + if accuracy is not None: + return accuracy + # If the accuracy is not provided, compute the accuracy using the accuracy + # function + return self.apply_fn(predictions, targets) + + +class NTKTrace(BaseMeasurement): + """ + Measurement class to record the trace of the NTK. + + Neural State Keys + ----------------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + """ + + def __init__( + self, + name: str = "ntk_trace", + rank: int = 1, + normalize: bool = True, + ): + """ + Constructor method of the NTKTrace class. + + Parameters + ---------- + name : str (default="ntk_trace") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=1) + The rank of the measurement, defining the tensor order of the + measurement. + public : bool (default=False) + Boolean flag to indicate whether the measurement resutls will be + accessible via a public attribute of the recorder. + normalize : bool (default=True) + Boolean flag to indicate whether the trace of the NTK will be normalized + by the size of the NTK matrix. + """ + super().__init__(name, rank) + self.normalise = normalize + + def apply(self, ntk: np.ndarray) -> np.ndarray: + """ + Method to compute the trace of the NTK. + + Parameters + ---------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + + Returns + ------- + np.ndarray + The trace of the NTK + """ + if ntk.shape[0] != ntk.shape[1]: + raise ValueError( + "To compute the trace of the NTK, the NTK matrix must" + f" be a square matrix, but got a matrix of shape {ntk.shape}." + ) + if len(ntk.shape) != 2: + raise ValueError( + "To compute the trace of the NTK, the NTK matrix must" + f" be a tensor of rank 2, but got a tensor of rank {len(ntk.shape)}." + ) + return compute_trace(ntk, normalize=self.normalise) + + +class NTKEntropy(BaseMeasurement): + """ + Measurement class to record the entropy of the NTK. + + Neural State Keys + ----------------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + """ + + def __init__( + self, + name: str = "ntk_cross_entropy", + rank: int = 1, + normalize_eigenvalues: bool = True, + effective: bool = False, + ): + """ + Constructor method of the NTKCrossEntropy class. + + Parameters + ---------- + name : str (default="ntk_trace") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=1) + The rank of the measurement, defining the tensor order of the + measurement. + normalize_eigenvalues : bool (default=True) + If true, the eigenvalues are scaled to look like probabilities. + effective : bool (default=False) + If true, the entropy is divided by the theoretical maximum entropy of + the system thereby returning the effective entropy / entropy density. + + """ + super().__init__(name=name, rank=rank) + self.normalize_eigenvalues = normalize_eigenvalues + self.effective = effective + + def apply(self, ntk: np.ndarray) -> np.ndarray: + """ + Method to compute the entropy of the NTK. + + Parameters + ---------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + Note that the NTK matrix needs to be a 2D square matrix. In case of a + 4D NTK tensor, please flatten the tensor to a 2D matrix before passing + it to this method. A default implementation of a flattenig method is + provided in the papyrus.utils.matrix_utils module as + flatten_rank_4_tensor. + + Returns + ------- + np.ndarray + The entropy of the NTK. + """ + # Assert that the NTK is a square matrix + if ntk.shape[0] != ntk.shape[1]: + raise ValueError( + "To compute the entropy of the NTK, the NTK matrix must" + f" be a square matrix, but got a matrix of shape {ntk.shape}." + ) + if len(ntk.shape) != 2: + raise ValueError( + "To compute the entropy of the NTK, the NTK matrix must" + f" be a tensor of rank 2, but got a tensor of rank {len(ntk.shape)}." + ) + # Compute the von Neumann entropy of the NTK + return compute_von_neumann_entropy( + ntk, + effective=self.effective, + normalize_eig=self.normalize_eigenvalues, + ) + + +class NTKSelfEntropy(BaseMeasurement): + """ + Measurement class to record the entropy of the diagonal of the NTK. + + This measurement can be interpreted as the entropy of the self-correlation of data + in a neural network. + + Neural State Keys + ----------------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + """ + + def __init__( + self, + name: str = "ntk_self_entropy", + rank: int = 0, + effective: bool = False, + ): + """ + Constructor method of the NTKSelfEntropy class. + + Parameters + ---------- + name : str (default="ntk_magnitude_distribution") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=1) + The rank of the measurement, defining the tensor order of the + measurement. + effective : bool (default=False) + Boolean flag to indicate whether the self-entropy of the NTK will be + normalized by the theoretical maximum entropy of the system. + """ + super().__init__(name, rank) + self.effective = effective + + def apply(self, ntk: np.ndarray) -> np.ndarray: + """ + Method to compute the self-entropy of the NTK. + + Parameters + ---------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + + Returns + ------- + np.ndarray + Self-entropy of the NTK. + """ + if ntk.shape[0] != ntk.shape[1]: + raise ValueError( + "To compute the self-entropy of the NTK, the NTK matrix must" + f" be a square matrix, but got a matrix of shape {ntk.shape}." + ) + if len(ntk.shape) != 2: + raise ValueError( + "To compute the self-entropy of the NTK, the NTK matrix must" + f" be a tensor of rank 2, but got a tensor of rank {len(ntk.shape)}." + ) + distribution = compute_gramian_diagonal_distribution(gram_matrix=ntk) + return compute_shannon_entropy(distribution, effective=self.effective) + + +class NTKMagnitudeDistribution(BaseMeasurement): + """ + Measurement class to record the magnitude distribution of the NTK. + + Note + ---- + This measurement is not applicable to varying number of inputs as its output size + depends on the number of inputs it is applied to. + + Measurements that return arrays with sizes that depend on the number of inputs + **cannot** be applied on varying number of inputs. This is because the number of + dimensions of the input need to be same for all subsequent calls, otherwise an error + will be raised when storing the results in the database. + + + Neural State Keys + ----------------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + """ + + def __init__( + self, + name: str = "ntk_magnitude_distribution", + rank: int = 0, + ): + """ + Constructor method of the NTKMagnitudeDistribution class. + + Parameters + ---------- + name : str (default="ntk_magnitude_distribution") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=1) + The rank of the measurement, defining the tensor order of the + measurement. + public : bool (default=False) + Boolean flag to indicate whether the measurement resutls will be + accessible via a public attribute of the recorder. + """ + super().__init__(name, rank) + + def apply(self, ntk: np.ndarray) -> np.ndarray: + """ + Method to compute the magnitude distribution of the NTK. + + Parameters + ---------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + + Returns + ------- + np.ndarray + The magnitude distribution of the NTK + """ + if ntk.shape[0] != ntk.shape[1]: + raise ValueError( + "To compute the magnitude distribution of the NTK, the NTK matrix must" + f" be a square matrix, but got a matrix of shape {ntk.shape}." + ) + if len(ntk.shape) != 2: + raise ValueError( + "To compute the magnitude distribution of the NTK, the NTK matrix must" + f" be a tensor of rank 2, but got a tensor of rank {len(ntk.shape)}." + ) + return compute_gramian_diagonal_distribution(gram_matrix=ntk) + + +class NTKEigenvalues(BaseMeasurement): + """ + Measurement class to record the eigenvalues of the NTK. + + Note + ---- + This measurement is not applicable to varying number of inputs as its output size + depends on the number of inputs it is applied to. + + Measurements that return arrays with sizes that depend on the number of inputs + **cannot** be applied on varying number of inputs. This is because the number of + dimensions of the input need to be same for all subsequent calls, otherwise an error + will be raised when storing the results in the database. + + Neural State Keys + ----------------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + """ + + def __init__( + self, + name: str = "ntk_eigenvalues", + rank: int = 1, + normalize: bool = True, + ): + """ + Constructor method of the NTKEigenvalues class. + + Parameters + ---------- + name : str (default="ntk_eigenvalues") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=1) + The rank of the measurement, defining the tensor order of the + measurement. + normalize : bool (default=True) + Boolean flag to indicate whether the eigenvalues of the NTK will be + normalized by the size of the NTK matrix. + """ + super().__init__(name, rank) + self.normalize = normalize + + def apply(self, ntk: np.ndarray) -> np.ndarray: + """ + Method to compute the eigenvalues of the NTK. + + Parameters + ---------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + + Returns + ------- + np.ndarray + The eigenvalues of the NTK + """ + if ntk.shape[0] != ntk.shape[1]: + raise ValueError( + "To compute the eigenvalues of the NTK, the NTK matrix must" + f" be a square matrix, but got a matrix of shape {ntk.shape}." + ) + if len(ntk.shape) != 2: + raise ValueError( + "To compute the eigenvalues of the NTK, the NTK matrix must" + f" be a tensor of rank 2, but got a tensor of rank {len(ntk.shape)}." + ) + return compute_hermitian_eigensystem(ntk, normalize=self.normalize)[0] + + +class NTK(BaseMeasurement): + """ + Measurement class to record the Neural Tangent Kernel (NTK). + + Note + ---- + This measurement is not applicable to varying number of inputs as its output size + depends on the number of inputs it is applied to. + + Measurements that return arrays with sizes that depend on the number of inputs + **cannot** be applied on varying number of inputs. This is because the number of + dimensions of the input need to be same for all subsequent calls, otherwise an error + will be raised when storing the results in the database. + + Neural State Keys + ----------------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + """ + + def __init__( + self, + name: str = "ntk", + rank: int = 2, + ): + """ + Constructor method of the NTK class. + + Parameters + ---------- + name : str (default="ntk") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=2) + The rank of the measurement, defining the tensor order of the + measurement. + """ + super().__init__(name, rank) + + def apply(self, ntk: np.ndarray) -> np.ndarray: + """ + Method to record the Neural Tangent Kernel (NTK). + + Parameters need to be provided as keyword arguments. + + Parameters + ---------- + ntk : np.ndarray + The Neural Tangent Kernel (NTK) matrix. + + Returns + ------- + np.ndarray + The Neural Tangent Kernel (NTK) matrix. + """ + return ntk + + +class LossDerivative(BaseMeasurement): + """ + Measurement class to record the derivative of the loss with respect to the neural + network outputs. + + Neural State Keys + ----------------- + loss_derivative : np.ndarray + The derivative of the loss with respect to the weights. + """ + + def __init__( + self, + apply_fn: Callable, + name: str = "loss_derivative", + rank: int = 1, + ): + """ + Constructor method of the LossDerivative class. + + Parameters + ---------- + apply_fn : Callable + The function to compute the derivative of the loss with respect to the + neural network outputs. + name : str (default="loss_derivative") + The name of the measurement, defining how the instance in the database + will be identified. + rank : int (default=1) + The rank of the measurement, defining the tensor order of the + measurement. + """ + super().__init__(name, rank) + self.apply_fn = apply_fn + + def apply(self, predictions: np.ndarray, targets: np.ndarray) -> np.ndarray: + """ + Method to record the derivative of the loss with respect to the neural network + outputs. + + Parameters need to be provided as keyword arguments. + + Parameters + ---------- + predictions : np.ndarray + The predictions of the neural network. + targets : np.ndarray + The target values of the neural network. + + Returns + ------- + np.ndarray + The derivative of the loss with respect to the neural network outputs. + """ + return self.apply_fn(predictions, targets) diff --git a/papyrus/neural_state/__init__.py b/papyrus/neural_state/__init__.py new file mode 100644 index 0000000..dc36950 --- /dev/null +++ b/papyrus/neural_state/__init__.py @@ -0,0 +1,30 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +from papyrus.neural_state.neural_state import NeuralState +from papyrus.neural_state.neural_state_creator import NeuralStateCreator + +__all__ = [ + NeuralState.__name__, + NeuralStateCreator.__name__, +] diff --git a/papyrus/neural_state/neural_state.py b/papyrus/neural_state/neural_state.py new file mode 100644 index 0000000..2eefe53 --- /dev/null +++ b/papyrus/neural_state/neural_state.py @@ -0,0 +1,73 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +from dataclasses import dataclass +from typing import List, Optional + +import numpy as np + + +@dataclass +class NeuralState: + """ + Data class to represent the state of a neural network. + + A neural network state can be represented in various ways. NeuralState offers a + structured solution to represent the state of a neural network in terms of different + properties. + If the default properties are not sufficient, the user can extend this class to + include more. In general, a property of a neural state can be any type of data, as + long as it is formatted as `List[Any]` or `np.array[Any]`. + + Attributes + ---------- + loss: Optional[List[np.ndarray]] + The loss of a neural network. + accuracy: Optional[List[np.ndarray]] + The accuracy of a neural network. + predictions: Optional[List[np.ndarray]] + The predictions of a neural network. + targets: Optional[List[np.ndarray]] + The targets of a neural network. + ntk: Optional[List[np.ndarray]] + The neural tangent kernel of a neural network. + """ + + loss: Optional[List[np.ndarray]] = None + accuracy: Optional[List[np.ndarray]] = None + predictions: Optional[List[np.ndarray]] = None + targets: Optional[List[np.ndarray]] = None + ntk: Optional[List[np.ndarray]] = None + + def get_dict(self) -> dict: + """ + Get a dictionary representation of the neural state. + + Only return the properties that are not None. + + Returns + ------- + dict + A dictionary representation of the neural state. + """ + return {k: v for k, v in self.__dict__.items() if v is not None} diff --git a/papyrus/neural_state/neural_state_creator.py b/papyrus/neural_state/neural_state_creator.py new file mode 100644 index 0000000..59727e4 --- /dev/null +++ b/papyrus/neural_state/neural_state_creator.py @@ -0,0 +1,88 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +from papyrus.neural_state.neural_state import NeuralState + + +class NeuralStateCreator: + """ + Class creating a neural state. + + The NeuralStateCreator class serves as instance mapping data and parameter state to + a NeuralState instance using a set of apply functions. The apply functions. + These apply functions are e.g. the neural network forward pass or the neural tangent + kernel computation. + + Attributes + ---------- + apply_fns : dict + A dictionary of apply functions that map the data and parameter state to a + NeuralState instance. + """ + + def __init__(self, network_apply_fn: callable, ntk_apply_fn: callable): + """ + Initialize the NeuralStateCreator instance. + + Parameters + ---------- + network_apply_fn : callable + The apply function that maps the data and parameter state to a + NeuralState instance. + ntk_apply_fn : callable + The apply function that maps the data and parameter state to a + NeuralState instance. + """ + self.apply_fns = { + "predictions": network_apply_fn, + "ntk": ntk_apply_fn, + } + + def __call__(self, params: dict, data: dict, **kwargs) -> NeuralState: + """ + Call the NeuralStateCreator instance. + + Parameters + ---------- + params : dict + A dictionary of parameters that are used in the apply functions. + data : dict + A dictionary of data that is used in the apply functions. + kwargs : Any + Additional keyword arguments that are directly added to the + neural state. + + Returns + ------- + NeuralState + The neural state that is created by the apply functions. + """ + neural_state = NeuralState() + + for key, apply_fn in self.apply_fns.items(): + neural_state.__setattr__(key, apply_fn(params, data)) + + for key, value in kwargs.items(): + neural_state.__setattr__(key, value) + + return neural_state diff --git a/papyrus/utils/__init__.py b/papyrus/utils/__init__.py new file mode 100644 index 0000000..cf60541 --- /dev/null +++ b/papyrus/utils/__init__.py @@ -0,0 +1,48 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +from papyrus.utils.analysis_utils import ( + compute_shannon_entropy, + compute_trace, + compute_von_neumann_entropy, +) +from papyrus.utils.matrix_utils import ( + compute_gramian_diagonal_distribution, + compute_hermitian_eigensystem, + compute_l_pq_norm, + flatten_rank_4_tensor, + normalize_gram_matrix, + unflatten_rank_4_tensor, +) + +__all__ = [ + compute_hermitian_eigensystem.__name__, + normalize_gram_matrix.__name__, + compute_gramian_diagonal_distribution.__name__, + compute_l_pq_norm.__name__, + flatten_rank_4_tensor.__name__, + unflatten_rank_4_tensor.__name__, + compute_trace.__name__, + compute_shannon_entropy.__name__, + compute_von_neumann_entropy.__name__, +] diff --git a/papyrus/utils/analysis_utils.py b/papyrus/utils/analysis_utils.py new file mode 100644 index 0000000..58e4636 --- /dev/null +++ b/papyrus/utils/analysis_utils.py @@ -0,0 +1,108 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +""" + +import numpy as np + +from papyrus.utils.matrix_utils import compute_hermitian_eigensystem + + +def compute_trace(matrix: np.ndarray, normalize: bool = False) -> np.ndarray: + """ + Compute the trace of a matrix, including optional normalization. + + Parameters + ---------- + matrix : np.ndarray + Matrix to calculate the trace of. + normalize : bool (default=True) + If true, the trace is normalized by the size of the matrix. + + Returns + ------- + trace : np.ndarray + Trace of the matrix. + """ + normalization_factor = np.shape(matrix)[0] if normalize else 1 + return np.trace(matrix) / normalization_factor + + +def compute_shannon_entropy(dist: np.ndarray, effective: bool = False) -> float: + """ + Compute the Shannon entropy of a given probability distribution. + + The Shannon entropy of a given probability distribution is computed using a + mask to neglect encountered zeros in the logarithm. + + Parameters + ---------- + dist : np.ndarray + Array to calculate the entropy of. + effective : bool (default = False) + If true, the Shannon entropy is normalized by re-scaling to the maximum + entropy. The method will return a value between 0 and 1. + + Returns + ------- + Entropy of the distribution + """ + mask = np.nonzero(dist) + scaled_values = -1 * dist[mask] * np.log(dist[mask]) + entropy = scaled_values.sum() + + if effective: + scale_factor = np.log(len(dist)) + entropy /= scale_factor + + return entropy + + +def compute_von_neumann_entropy( + matrix: np.ndarray, effective: bool = True, normalize_eig: bool = True +) -> float: + """ + Compute the von-Neumann entropy of a matrix. + + Parameters + ---------- + matrix : np.ndarray + Matrix for which the entropy should be computed. + effective : bool (default=True) + If true, the entropy is divided by the theoretical maximum entropy of + the system thereby returning the effective entropy. + normalize_eig : bool (default = True) + If true, the eigenvalues are scaled to look like probabilities. + + Returns + ------- + entropy : float + Von-Neumann entropy of the matrix. + """ + eigvals, _ = compute_hermitian_eigensystem(matrix, normalize=normalize_eig) + + entropy = compute_shannon_entropy(eigvals) + + if effective: + maximum_entropy = np.log(len(eigvals)) + entropy /= maximum_entropy + + return entropy diff --git a/papyrus/utils/matrix_utils.py b/papyrus/utils/matrix_utils.py new file mode 100644 index 0000000..b56dbf7 --- /dev/null +++ b/papyrus/utils/matrix_utils.py @@ -0,0 +1,235 @@ +""" +papyrus: a lightweight Python library to record neural learning. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Summary +------- +Matrix utils for papyrus. +""" + +import logging + +import numpy as np + +logger = logging.getLogger(__name__) + + +def compute_hermitian_eigensystem( + matrix: np.ndarray, normalize: bool = False, clip: bool = True +): + """ + Compute the eigenspace of a hermitian matrix. + + Parameters + ---------- + matrix : np.ndarray + Matrix for which the space should be computed. Must be hermitian. + normalize : bool (default=False) + If true, the eigenvalues are divided by the sum of the eigenvalues of the + given matrix. This is equivalent to dividing by the size of the dataset. + clip : bool (default=True) + Clip the eigenvalues to a very small number to avoid negatives. This should + only be used if you are sure that negative numbers only arise due to some + numeric reason and that they should not exist. + + Returns + ------- + eigenvalues : np.ndarray + Eigenvalues of the matrix. The eigenvalues are sorted in descending order. + eigenvectors : np.ndarray + Eigenvectors of the matrix. + The column `eigenvectors[:, i]` is the normalized eigenvector corresponding + to the eigenvalue `eigenvalues[i]`. + """ + eigenvalues, eigenvectors = np.linalg.eigh(matrix) + + if clip: + # logger.warning("Eigenvalues are being clipped to avoid negative values.") + eigenvalues = np.clip(eigenvalues, 1e-14, None) + + if normalize: + eigenvalues /= eigenvalues.sum() + + return eigenvalues[::-1], eigenvectors[:, ::-1] + + +def normalize_gram_matrix(gram_matrix: np.ndarray): + """ + Method for normalizing a gram matrix. + + The normalization is done by dividing each element of the gram matrix by the + square root of the product of the corresponding diagonal elements. This is + equivalent to normalizing the inputs of the gram matrix. + + Parameters + ---------- + gram_matrix : np.ndarray + Gram matrix to normalize. + + Returns + ------- + normalized_gram_matrix : np.ndarray + A normalized gram matrix, i.e, the matrix given if all of its inputs + had been normalized. + """ + order = np.shape(gram_matrix)[0] + + diagonals = np.diagonal(gram_matrix) + + repeated_diagonals = np.repeat(diagonals[None, :], order, axis=0) + + normalizing_matrix = np.sqrt(repeated_diagonals * repeated_diagonals.T) + + # Check for zeros in the normalizing matrix + normalizing_matrix = np.where( + normalizing_matrix == 0, 0, 1 / normalizing_matrix + ) # Avoid division by zero + + return gram_matrix * normalizing_matrix + + +def compute_gramian_diagonal_distribution(gram_matrix: np.ndarray) -> np.ndarray: + """ + Compute the normalized distribution of the diagonals of a gram matrix. + + The distribution is computed by taking the square root of the diagonal elements + of the gram matrix and normalizing them by the sum. + This method is equivalent to the distribution of the magnitudes of the vectors + that were used to compute the gram matrix. + + Parameters + ---------- + gram_matrix : np.ndarray + Gram matrix to compute the diagonal distribution of. + + Returns + ------- + magnitude_density: np.ndarray + Magnitude density of the individual entries. + """ + magnitudes = np.sqrt(np.diagonal(gram_matrix)) + distribution = magnitudes / magnitudes.sum() + return distribution + + +def compute_l_pq_norm(matrix: np.ndarray, p: int = 2, q: int = 2): + """ + Compute the L_pq norm of a matrix. + + The norm calculates (sum_j (sum_i abs(a_ij)^p)^(p/q) )^(1/q) + For the defaults (p = 2, q = 2) the function calculates the Frobenius + (or Hilbert-Schmidt) norm. + + Parameters + ---------- + matrix: np.ndarray + Matrix to calculate the L_pq norm of + p: int (default=2) + Inner power of the norm. + q: int (default=2) + Outer power of the norm. + + Returns + ------- + calculate_l_pq_norm: np.ndarray + L_qp norm of the matrix. + """ + inner_sum = np.sum(np.power(np.abs(matrix), p), axis=-1) + outer_sum = np.sum(np.power(inner_sum, q / p), axis=-1) + return np.power(outer_sum, 1 / q) + + +def flatten_rank_4_tensor(tensor: np.ndarray) -> np.ndarray: + """ + Flatten a rank 4 tensor to a rank 2 tensor using a specific reshaping. + + With this function a tensor of shape (k, l, m, n) is reshaped to a tensor of shape + (k * m, l * n). The reshaping is done by concatenating axes 1 and 2, and + then 3 and 4. + + Parameters + ---------- + tensor : np.ndarray (shape=(k, l, m, n)) + Tensor to flatten. + + Returns + ------- + A 2-tuple of the following form: + + flattened_tensor : np.ndarray (shape=(k * m, l * n)) + Flattened tensor. + shape : tuple + Shape of the original tensor. + """ + # Check if the tensor is of rank 4 + if len(tensor.shape) != 4: + raise ValueError( + "The tensor is not of rank 4. " + f"Expected rank 4 but got {len(tensor.shape)}." + ) + shape = tensor.shape + _tensor = np.moveaxis(tensor, [1, 2], [2, 1]) + flattened_tensor = _tensor.reshape(shape[0] * shape[2], shape[1] * shape[3]) + return flattened_tensor, shape + + +def unflatten_rank_4_tensor(tensor: np.ndarray, new_shape: tuple) -> np.ndarray: + """ + Unflatten a rank 2 tensor to a rank 4 tensor using a specific reshaping. + + This is the inverse operation of the flatten_rank_4_tensor function. + The tensor is assumed to be of shape (k * m, l * n) and is reshaped to a tensor + of shape (k, l, m, n). The reshaping is done by splitting the first axis into + two axes, and then the second axis into two axes. + + Parameters + ---------- + tensor : np.ndarray (shape=(k * m, l * n)) + Tensor to unflatten. + shape : tuple + Shape of the original tensor. Must be of rank 4. + + Returns + ------- + unflattened_tensor : np.ndarray (shape=(k, l, m, n)) + Unflattened tensor. + """ + # Check if the tensor is of rank 2 + if len(tensor.shape) != 2: + raise ValueError( + "The tensor is not of rank 2. " + f"Expected rank 2 but got {len(tensor.shape)}." + ) + # Check if the shape is of rank 4 + if len(new_shape) != 4: + raise ValueError( + "The shape is not of rank 4. " f"Expected rank 4 but got {len(new_shape)}." + ) + # Check if the shapes match + if not new_shape[0] * new_shape[2] == tensor.shape[0]: + raise ValueError( + "The shape of the tensor does not match the given dimensions. " + f"Expected {new_shape[0] * new_shape[2]} but got {tensor.shape[0]}." + ) + if not new_shape[1] * new_shape[3] == tensor.shape[1]: + raise ValueError( + "The shape of the tensor does not match the given dimensions. " + f"Expected {new_shape[1] * new_shape[3]} but got {tensor.shape[1]}." + ) + _tensor = tensor.reshape(new_shape[0], new_shape[2], new_shape[1], new_shape[3]) + return np.moveaxis(_tensor, [2, 1], [1, 2]) diff --git a/pyproject.toml b/pyproject.toml index 0479371..bf1881f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ description = "Like our ancestors started capturing their thoughts on papyrus, w readme = "README.md" requires-python = ">=3.11" keywords = ["neural networks", "artificial intelligence"] -license = {text = "BSD-3-Clause"} +license = {file = "LICENSE"} version = "0.0.1" + +[tool.pytest.ini_options] +pythonpath = ["papyrus",] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 3827286..2498c9e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,7 @@ install_requires = [options.extras_require] dev = + numpy>=1.26.0 isort>=5.13.2 black>=24.4.0 pytest>=8.1.1 diff --git a/test.ipynb b/test.ipynb new file mode 100644 index 0000000..8b7f94e --- /dev/null +++ b/test.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/aElEQVR4nO3de1iU953//9dwGkBgOMkgAuIpGqJCImLMYasNrSWtaZK26/abbYnZ2qtbzCZLT7rdmG9/29TuZutltzu7Xu1+re325Kab2DamNg05mKQmKgajMR6IRFHkJMLAAAPM3L8/kDHUE8jM3DPD83Fdc3Vn5mbu99ztZl753O/P52MxDMMQAABAmIgyuwAAAICxILwAAICwQngBAABhhfACAADCCuEFAACEFcILAAAIK4QXAAAQVggvAAAgrMSYXYC/eb1eNTY2Kjk5WRaLxexyAADAKBiGoa6uLuXk5Cgq6upjKxEXXhobG5WXl2d2GQAA4Do0NDQoNzf3qsdEXHhJTk6WNPTlU1JSTK4GAACMhtPpVF5enu93/GoiLrwM3ypKSUkhvAAAEGZG0/IRMQ27DodDhYWFWrRokdmlAACAALJE2q7STqdTNptNnZ2djLwAABAmxvL7HTEjLwAAYGIgvAAAgLASMeGFnhcAACYGel4AAIDp6HkBAAARi/ACAADCCuEFAACEFcILAAAIKxETXphtBADAxMBsIwAAYLqx/H5H3MaMAAAgMN4+3aFf15zWnOxkPbB4mml1RMxtIwAAEFhHm7r0090n9fw7zabWQXgBAACj4nIPSpKSrObeuCG8AACAUXH1eyRJk6zRptZBeAEAAKPSfWHkZRIjLwAAIBxw28jPWOcFAIDAYuTFzyorK3X48GHt3bvX7FIAAIhILsILAAAIJy73UMNuEg27AAAgHPhuG8Ux8gIAAMIADbsAACCs0PMCAADCysXZRvS8AACAEGcYxgdW2GXkBQAAhDj3oFceryGJ8HKJjo4OlZSUqLi4WPPmzdOPfvQjs0sCAGDCG+53kcyfbWTu2S8jOTlZu3btUmJiolwul+bNm6f7779fGRkZZpcGAMCENbzGS0JstKKjLKbWEnIjL9HR0UpMTJQkud1uGYYhwzBMrgoAgIktVLYGkAIQXnbt2qUVK1YoJydHFotF27dvv+QYh8OhgoICxcfHa/HixdqzZ8+I9zs6OlRUVKTc3Fx97WtfU2Zmpr/LBAAAY+DqH17jxdyZRlIAwovL5VJRUZEcDsdl39+2bZuqqqr0+OOPa//+/SoqKtLy5cvV0tLiOyY1NVUHDhxQfX29fvGLX6i5ufmK53O73XI6nSMeAADAvyJ65KW8vFzf/va3dd999132/Y0bN2r16tVatWqVCgsLtXnzZiUmJmrLli2XHGu321VUVKRXX331iufbsGGDbDab75GXl+e37wIAAIaEygJ1UpB7Xvr7+1VTU6OysrKLBURFqaysTLt375YkNTc3q6urS5LU2dmpXbt2ac6cOVf8zHXr1qmzs9P3aGhoCOyXAABgAgqVrQGkIM82amtrk8fjkd1uH/G63W7XkSNHJEknT57UF7/4RV+j7sMPP6z58+df8TOtVqusVmtA6wYAYKLrdofGAnVSCE6VLi0tVW1t7Zj/zuFwyOFwyOPx+L8oAAAmuIsjLxHYsHs1mZmZio6OvqQBt7m5WdnZ2eP67MrKSh0+fFh79+4d1+cAAIBL+XpeTF6gTgpyeImLi9PChQtVXV3te83r9aq6ulpLliwZ12c7HA4VFhZq0aJF4y0TAAD8mVCabeT3Crq7u1VXV+d7Xl9fr9raWqWnpys/P19VVVWqqKhQSUmJSktLtWnTJrlcLq1atWpc562srFRlZaWcTqdsNtt4vwYAAPiAiG7Y3bdvn5YtW+Z7XlVVJUmqqKjQ1q1btXLlSrW2tmr9+vVqampScXGxdu7ceUkTLwAACB0R3bC7dOnSay7nv2bNGq1Zs8av56VhFwCAwLm4zssEa9gNJBp2AQAInOHtASZcwy4AAAhPE3aF3UBithEAAIHjutDzEgoNuxETXrhtBABA4NDzAgAAwoZhGL6eF0ZeAABAyOsd8Mh7YSIxPS9+RM8LAACBMby6rsUiJcZx28hv6HkBACAwhpt1J8XFyGKxmFxNBIUXAAAQGKHUrCsRXgAAwDWE0qaMUgSFF3peAAAIjFDalFGKoPBCzwsAAIHhG3kJga0BpAgKLwAAIDBcIbSjtER4AQAA13DxthENuwAAIAzQsAsAAMJKKO0oLUVQeGG2EQAAgeHqv7hIXSiImPDCbCMAAAKDReoAAEBYYZ0XAAAQVmjYBQAAYcXVz8gLAAAIIyxSBwAAwko3DbsAACCc0LAbIKzzAgCA/3m9hnr6uW0UEKzzAgCA/w0360qMvAAAgDAw3KwbHWWRNSY0YkNoVAEAAEKSr1k3LloWi8XkaoYQXgAAwBWFWrOuRHgBAABXEWo7SkuEFwAAcBWhtjWARHgBAABXMTzbKFQWqJMILwAA4Cp8WwPEMfJyRQ0NDVq6dKkKCwu1YMECPfXUU2aXBADAhBWKDbuhU8kFMTEx2rRpk4qLi9XU1KSFCxfq7rvv1qRJk8wuDQCACScUG3ZDp5ILpkyZoilTpkiSsrOzlZmZqfb2dsILAAAm6A6xHaWlANw22rVrl1asWKGcnBxZLBZt3779kmMcDocKCgoUHx+vxYsXa8+ePZf9rJqaGnk8HuXl5fm7TAAAMAoXbxtFcMOuy+VSUVGRHA7HZd/ftm2bqqqq9Pjjj2v//v0qKirS8uXL1dLSMuK49vZ2ff7zn9cPf/hDf5cIAABGqbt/Atw2Ki8vV3l5+RXf37hxo1avXq1Vq1ZJkjZv3qwdO3Zoy5YtWrt2rSTJ7Xbr3nvv1dq1a3Xbbbdd9Xxut1tut9v33Ol0+uFbAAAAKTR7XoI626i/v181NTUqKyu7WEBUlMrKyrR7925JkmEYevDBB/XhD39Yn/vc5675mRs2bJDNZvM9uMUEAID/hOJso6CGl7a2Nnk8Htnt9hGv2+12NTU1SZJef/11bdu2Tdu3b1dxcbGKi4t18ODBK37munXr1NnZ6Xs0NDQE9DsAADCRhGLDbuhUcsEdd9whr9c76uOtVqusVqscDoccDoc8Hk8AqwMAYGKZEA27V5OZmano6Gg1NzePeL25uVnZ2dnj+uzKykodPnxYe/fuHdfnAACAiyZ8z0tcXJwWLlyo6upq32ter1fV1dVasmTJuD7b4XCosLBQixYtGm+ZAADgAt/GjCG0PYDfK+nu7lZdXZ3veX19vWpra5Wenq78/HxVVVWpoqJCJSUlKi0t1aZNm+RyuXyzj65XZWWlKisr5XQ6ZbPZxvs1AACY8AY9XrkHh1o5Qqlh1++V7Nu3T8uWLfM9r6qqkiRVVFRo69atWrlypVpbW7V+/Xo1NTWpuLhYO3fuvKSJFwAAmGt4U0YptG4b+b2SpUuXyjCMqx6zZs0arVmzxq/npWEXAAD/Gl6gLi46SnExobOXc+hUMk407AIA4F89F/pdEkNoppEUQeEFAAD4Vyg260oRFF6YbQQAgH8N97yEUrOuFEHhhdtGAAD4l2/khdtGAAAgHITiAnUS4QUAAFyBqz/0NmWUIii80PMCAIB/dTPyElj0vAAA4F8XN2UkvAAAgDAwPNuIhl0AABAWuG0UYPS8AADgX9w2CjB6XgAA8C9W2AUAAGGFdV4AAEBYCdXtAUKrGgAAYLq+AY+ON3frnKtfUujNNiK8AAAwwQx6vHr3bJdOtrt0vmdAnT39Ot8zoNYut442damutVser+E7Pi0xzsRqLxUx4cXhcMjhcMjj8ZhdCgAAIaWrb0CHG53a+3673qxv1/6T5+Xqv/rvZWpirG7KSdHSG7JUkDkpSJWOjsUwDOPah4UPp9Mpm82mzs5OpaSkmF0OAABBYxiGTp/v1YHTHTrc6NTRpi4daerSmY7eS45NiY/RnOxkpSXGKS0xTqmTYpWWGKfZWUkqzElRdkq8LBZL0Gofy+93xIy8AAAwEXi8htpd/Wrrdqut261z3f1qaO9RbUOHDpzuUFt3/2X/LjslXgsL0lRakK7S6emaY09WVFTwwok/EV4AAAgDdS3d+tkbJ/W/+0+rq2/wisfFRlt045QUzZtq09zsZM2xJ2tOdrJSQ6xvZTwILwAAhCDDMNTtHtTrdW367zdO6vW6c773LJahJtrMpDhlJlmVnRKveVNtKspL1U05KYqPDa3ZQf5GeAEAwARdfQM6ea5HjR29OtPRq8aOXjV29KnZ2aeWLrdau9zqHbjYVBtlkT48167PL5mm22ZmKCZ64i7VRngBACCAvF5Dx1q6VHPyvI43d6uuZejR5Owb1d9PTrbq0wtz9cDifOWmJQa42vBAeAEAwA8Mw5Czd1Ct3X1q7erXO42derO+XXvfb1dHz8Bl/yZ9Upxy0xI0NTVBOalD/5lti9fkZKuykq2anGxVYojtKxQKuCIAAIzBgMerE60uvXvWqXfPOnX4rFPvtXSrrbtf/R7vZf8mITZat0xLVeGUFM3KShp6TE6WLTE2yNVHhogJLyxSBwAYr/5Br9q63b6pyOe6+9XS5dap9h41tPfoZLtLjR19I1af/XMp8THKTLZqWnqiSqdnaPGMdM2falPsBO5R8TcWqQMATEgu96Dq21w6eKZTb5/u0IGGTh1r7tLgVYLJsGRrjOZOSdbc7BTdOCVFc7KTlG1LUMakuIif6RMoLFIHAJjQvF5Drd1DIyYnz/Xo1DmXTrX3qOnCTJ4Wp1vd7suvlRITZVFGUpwyJlmVcWEqcl56ovLTEzUtI1HT0hM1Odka1NVnMRLhBQAQNgzD8N3Gab0wnbit2+37v1s+8NqoRlDiYzR/qk0LclNVlGvT/FybpqYmEExCHOEFABCSBj1enWrv0bHmLh0649Shxk4dOuNUW7d7VH8fHWVRTmq8pqVPUn7G0MjJFFu8spLjlZVilT0lXklWfgbDEf+tAQBMZRiGznT06nDj0Myd483dOt7Spffbei47eyfKIuWmJfqmEmcmDT2yUi5OL85KjldGUhxNshGK8AIACLjO3gE1tPfobOfQCrLDj4b2Xh0+61Rn7+XXQYmPjdKsrCTdNMWmeVNTdNNUm27MTlFCHE2xExnhBQDgFx09/apvc6m+zaX321yqv9Aoe7K954qLtA2LibJoVlaSCnNSNDc7WbOzkjUrK0lTUxPCdudjBE5Ihpf77rtPL7/8su666y79+te/NrscAJjQ2rrdOu/ql7NvUF19A+rqG9S5brdOn+8denT0qKG994qjJ8Myk+KUk5oge0q87ClW2ZPjlW2L141TUjTbniRrDKMpGJ2QDC+PPPKIHnroIf3kJz8xuxQAmJAa2nu04+BZPft2ow6dcY7677JT4lWQmajpmUkqyBiaWpx/oWGW5lj4S0j+L2np0qV6+eWXzS4DACJau6tftQ3n1dEzNJrS1TcgZ9+g3qxv14GGDt9xFouUmhCrpPgYJVtjlRwf49uTJzctcWhvnrQE5acnsg8PgsLv/yvbtWuXnnzySdXU1Ojs2bN65plndO+99444xuFw6Mknn1RTU5OKior0gx/8QKWlpf4uBQBwgddrqHfAo7dPd+rV46169XibDjV26kprrEdZpFtnZOgTC3K0/Ca7MpKswS0YuAq/hxeXy6WioiI99NBDuv/++y95f9u2baqqqtLmzZu1ePFibdq0ScuXL9fRo0eVlZXl73IAIKIZhqFmp1vvtXbrVHuPTp8f6j85fb5HzU63evoH1TvgUd/A5TcMnJ2VpGxbvJI/MKpSkDlJy2/K1uRkAgtCk9/DS3l5ucrLy6/4/saNG7V69WqtWrVKkrR582bt2LFDW7Zs0dq1a8d8PrfbLbf74oJFTufo780CQLhp7OjVi0daVHPyvN5r7dZ7Ld1y9Y9+Q9rMJKvunJ2pO2dn6o5ZmcpKiQ9gtUBgBPXmZH9/v2pqarRu3Trfa1FRUSorK9Pu3buv6zM3bNigb33rW/4qEQBMNeDxqqtveLTEo95+jzp7B/Sn99r04pFWvXv20n9Bi46yKD89UQUZicpNS1Re+lAvyhTb0Aqy8bHRSoiLVkJstBLjoln6HmEvqOGlra1NHo9Hdrt9xOt2u11HjhzxPS8rK9OBAwfkcrmUm5urp556SkuWLLnsZ65bt05VVVW+506nU3l5eYH5AgDgJ509A3r+cJNePtaq5s4+tbv6dc7Vf83pxhaLdEt+mu6cnam52cmaOTlJ0zImKS6GlWQxcYRkW/gLL7ww6mOtVqusVqscDoccDoc8ntEPnwJAsPQPetXS1afd753TcwfP6rW6Ng14rrxxYFxMlBJioxUfO/Sf86badNeNWfrQDVlKnxQXxMqB0BPU8JKZmano6Gg1NzePeL25uVnZ2dnj+uzKykpVVlbK6XTKZrON67MAYKwMw1Brl1vHW7pVd+FR3+ZSs7NPrd3uy64wO8eerPL52brBnqz0SXHKTIpT+iSrbAmximZVWeCKghpe4uLitHDhQlVXV/umT3u9XlVXV2vNmjXj+mxGXgAEW/+gV7tPnNMf3mnSC4eb1dJ19d2OY6Mtmjk5SXfPn6K750/RrKykIFUKRBa/h5fu7m7V1dX5ntfX16u2tlbp6enKz89XVVWVKioqVFJSotLSUm3atEkul8s3++h6MfICIBA8XkM1J8/rbGevuvoG1e0eVHffoN4/59IrR1vV5R70HRtlkaZlTNLMyUmabU/SzMlJmmKL1+RkqyYnWZWaGEuzLOAHfg8v+/bt07Jly3zPh5tpKyoqtHXrVq1cuVKtra1av369mpqaVFxcrJ07d17SxAsAZjEMQ281dOi3tY3acfCsWq8yojI52aqPFtr10ZuytXh6uuJj2Z8HCDSLYVxpfcXw8sHbRseOHVNnZ6dSUlLMLgtACDIMQ03OPh1t6lJdS7fO9/Sru29QXe5BdfUN6kiTUw3tvb7jbQmxKpySouT4mAtL5McobVKc7pw9WTfnpbLrMeAHw3dORvP7HTHhZdhYvjyAyOdyD+pIU5fePevUu2edOtrUpaPNXerqG7zq3yXGResjhXbdU5SjO2dPZioyEGBj+f0OyanSADBWp871qPZ0hxrae3T6/NDy+Kfahx6X+1e06CiLpmdO0g32JGUlDy3mlhQfo+T4GGUlx+uOWZlKiOMWEBCKIia8MNsImHgMw9AbJ9r1/147oeojLVfcZDAr2aobp6RceCTrBnuyZkyeJGsM4QQIR9w2AhB2Wrr69OqxNm15vV7vNF5cLv+W/FQVZE4aWiI/bWiJ/Nn2JGWyIzIQ8rhtBCBsnXf1673WbnW7B4f29hnwqKffo1PnenT4Qt9KW3e/7/j42Ch96pZcPXTHdM2czLopwERAeAFgmm73oA6d6dTbpzv09ulOvX26U6fae675d1EWacbkJN1381T9n9J8pbFcPjChREx4oecFCG2DHq/ePdultxrO60DDUGCpa+2+bJ9KblqCUuJjlRg3tBtyfGy0ptjifX0rc+zJNNMCExg9LwD8orNnQA3ne9Tv8ap/cOjR0+/RO42d2vf+edU2dKh34NJ/ucixxWt+rk0LclNVlJuq+bk22RJiTfgGAMxEzwuAgDIMQ3Ut3dp38rz2nzyv/afO671W1zX/LiU+Rjfnp6koL1VFuTbNz7UpKzk+CBUDiCSEFwCjVtfSpd8eOKtnDzTqRNulYWVyslXxsVGKi45SXEy0rDFRmjk5SSUFaVo4LU2zJiexGi2AcYuY8ELPC+B/Lveg3jrVoT3vt+v5d5p0pKnL9541Jkq35KfplmmpuiU/TTfnpymdxlkAQUDPCwB5vYaau/r0fluPTp5z6Vhzt2pOtutQo1Me78V/RMRGW/QXsydrRVGOygrtSrJGzL//ADAZPS8Arqi336N3m5w6dKbzwsOp91q75R70Xvb4qakJKilI0+0zM/XRm+xKTWR0BYC5CC9ABGvp6tM7Z5w6fNbp25zwRGu3vFfY6ycvLUHTMiZpeuYkFeelatH0dE1NTQh+4QBwFYQXIEIYhqGG9l69WX9Oe99v1576dr1/7vILvmUmWTV/aormTbVp3lSb5tiTNTUtQbHR7JwMIPQRXoAIUN/m0jefOag/vXduxOsWizRrcpJunJKiuVOSdeOUFBVOSVFWslUWC7N+AISniAkvzDbCRDTg8eqHu07o+9XH1T/oVUyURUV5qSqdnq7S6elaOC1NKfEs+AYgsjDbCAhDhmHorYYO/cPTB33Tl++cnakn7p2v/IxEk6sDgLFjthEQYQzD0Ik2l/bUt+vNE+e0p75djZ19kqS0xFg99olC3XfzVG4FAZgQCC9AiHL2Dej14216+WirXjnWqiZn34j3Y6Is+mTxVH3z4zeyOByACYXwAoQAwzB0+nyv3j49tNvy/lPntf9Ux4gF4uJionRzXqoWT0/X4hkZujk/VYlx/L8wgImHf/IBJuh2D+pAQ4f2nzyvtxo6dKChQ+dc/ZccN2PyJH3ohslaOidLi6enKz422oRqASC0EF6AIKlr6dJvaxv1/OFmHW3u0p+3ysdGWzQ3O0Xzc20qyrVpyYxMmm8B4DIIL0AANbT36LmDZ/Wb2kYdPusc8V5uWoJuzk/TLfmpKs5L1Y1TUhhZAYBRILwAftTVN6A3TrTr1eOteu14m060uXzvxURZ9Bc3TNaKoim6fWamslLiTawUAMJXxIQXFqmDGQY9Xh043aFXj7fpteNtqm3o0OAHmmyjoywqmZamTxZPVfm8bKUxKwgAxo1F6oBRcA96dOhMp06f79WZjl6dOd+rhvO9euvkeXW5B0ccW5CRqDtnT9YdszO1ZGYGK9wCwCiwSB3gJ93uQf38jZP6r9fq1drlvuwxtoRY3T4rQ3fMmqw7Z2cqL50mWwAIJMILcBntrn5tfb1eW//0vpx9QyMrGZPiNDMrSbmpCZqalqCc1AQVThnamTk6ipVtASBYCC+Y8AY9Xh1v6dbbpzt04MIicUfOdvl6V2ZMnqS//dBMfbJ4quJiokyuFgBAeMGE0+0eVO2pDu19v101J8/rrVPn5eq/tNF7/lSbvrx0pj56UzYjKwAQQggvmBAGPF69fLRVv65p0ItHWjTgGdmnnmSN0fypNi3Is6koN1ULcm2amprARocAEIIIL4hYvf0eHT7r1B/eadLT+8+orftiw+3U1AQtKkjTwoJ0lUxL0w32ZEZXACBMhGR4efbZZ/WVr3xFXq9X3/jGN/SFL3zB7JIQBjp7BvTr/ad18HSH3ml06r3Wbn1gyRVlTIrTfTdP1WdK8jQnO9m8QgEA4xJy4WVwcFBVVVV66aWXZLPZtHDhQt13333KyMgwuzSEsOffadI3tx+6ZDpzZlKcFk5L06duydWyuVmKjabhFgDCXciFlz179uimm27S1KlTJUnl5eV6/vnn9dnPftbkyhCK2l39evy37+h3BxolSTMyJ+m+m6fqpqkpuinHpqxkK30rABBh/P6vobt27dKKFSuUk5Mji8Wi7du3X3KMw+FQQUGB4uPjtXjxYu3Zs8f3XmNjoy+4SNLUqVN15swZf5eJMNfTP6hf15zWRza+ot8daFSURfrSh2bquUfu1MN3zdaH59plT4knuABABPL7yIvL5VJRUZEeeugh3X///Ze8v23bNlVVVWnz5s1avHixNm3apOXLl+vo0aPKysrydzmIIL39Hr10tEU73j6rF4+0qHdgaHrzDfYkPfnpIhXlpZpbIAAgKPweXsrLy1VeXn7F9zdu3KjVq1dr1apVkqTNmzdrx44d2rJli9auXaucnJwRIy1nzpxRaWnpFT/P7XbL7b7Y5+B0Ov3wLRAqzrv69eKRFv3xcLNeOdbqCyySlJeeoJUleVr9FzNkjYk2sUoAQDAFteelv79fNTU1Wrdune+1qKgolZWVaffu3ZKk0tJSHTp0SGfOnJHNZtPvf/97PfbYY1f8zA0bNuhb3/pWwGtH8Ljcg3pqX4OeO9Skfe+3j5gxlJuWoI8vmKKPz5+i+VNt3BYCgAkoqOGlra1NHo9Hdrt9xOt2u11HjhwZKigmRt/73ve0bNkyeb1eff3rX7/qTKN169apqqrK99zpdCovLy8wXwAB1e7q19Y/va+f/Ol9dfYO+F6fm52sjxba9ZHCbM2bmkJgAYAJLuRmG0nSPffco3vuuWdUx1qtVlmtVjkcDjkcDnk8ly7zjtDW2NGrH716Qr/a0+C7LTQ9c5L++tZp+mihnV2aAQAjBDW8ZGZmKjo6Ws3NzSNeb25uVnZ29rg+u7KyUpWVlXI6nbLZbOP6LATH8eYubX7lhH5Te8a3CeK8qSn68tJZWs5+QgCAKwhqeImLi9PChQtVXV2te++9V5Lk9XpVXV2tNWvWjOuzGXkJH/tPndd/vPSeXnj3YohdMiNDX142U3fMyuS2EADgqvweXrq7u1VXV+d7Xl9fr9raWqWnpys/P19VVVWqqKhQSUmJSktLtWnTJrlcLt/so+vFyEvo6xvw6DvPvauf7j4pSbJYpOWF2frS0pkqZpozAGCU/B5e9u3bp2XLlvmeDzfTVlRUaOvWrVq5cqVaW1u1fv16NTU1qbi4WDt37rykiReR5UiTU3/3y7d0rLlbkvSpW3L1t0tnalZWksmVAQDCjcUwDOPah4W+D942OnbsmDo7O5WSkmJ2WROeYRj6yZ/e13d+f0T9g15lJln1r59ZoKVzWJAQAHDR8J2T0fx+R0x4GTaWL4/Aqmvp0v/97WG9VtcmSVo2Z7Ke/EyRMpOsJlcGAAg1Y/n9Dsmp0ghvzr4B/dsLx7X1T+9r0GsoLiZK37z7Rn1+yTSacQEA4xYx4YXZRuYb9Hj19P4z+pc/HFFbd78kqezGLD32iUJNy5hkcnUAgEjBbSOMW4uzT7/c06Bf7jmlJmefJGlG5iQ9tqJQy+htAQCMAreNEBT7T53X/3utXn841ORbZC5jUpy++BcztOr26YqLiTK5QgBAJCK8YMzq21z6598f0c53mnyvLZyWps8vmaaPzctmh2cAQEBFTHih5yXwznW79W/Vx/XzN09p0GsoyiLdf0uuVt1eoJtyWBgQABAc9LzgmrxeQz9/86T+eedRdbsHJQ1Ne1539426wZ5scnUAgEhAzwv8pqG9R1/79QG9caJd0tDGif9QfqNum5VpcmUAgImK8ILLGh5t2fD7I+rp9yghNlpry+fqc7dOUxS7PQMATER4wSX6Bjz64n/XaNexVklS6fR0PfnpBazVAgAICRETXmjY9Q/DMPSN/31bu461Kj42St/42FxVLClgtAUAEDJo2MUIP6g+ru/98Zhioiz66UOl9LYAAIJiLL/frCIGnx1vn9X3/nhMkvStT95EcAEAhCTCCyRJb5/u0FeeqpUkrbq9QA8snmZuQQAAXAHhBTrb2avVP92nvgGvls6ZrH/8eKHZJQEAcEURE14cDocKCwu1aNEis0sJK7UNHbrP8Sc1O926wZ6kH3z2ZkXTnAsACGE07E5g/7O3Qf+4/ZD6PV7NnDxJW1eVKi890eyyAAATECvs4qr6B736p2cP67/fOClJ+kihXRv/skjJ8bEmVwYAwLURXiYQwzD0et05fe+PR/XWqQ5J0t+X3aCHPzyLdVwAAGGD8DIBuAc9+t2Bs/qvV0/oSFOXJCnZGqNNf1Wsu260m1wdAABjQ3iJcC8cbtY/PHNQLV1uSVJiXLT+siRPX7hzunLT6G8BAIQfwksEa+nq099vq1WXe1D2FKsevG26/k9pvmyJ9LYAAMIX4SWCbXjuiLrcg1qQa9Ovv3Sb4mIiZmY8AGAC49csQu1+75yeeeuMLBbp2/fOI7gAACJGxPyisUjdRf2DXq3/zSFJ0gOL87UgN9XcggAA8CMWqYtAm195T9/9/RFlTIrTi19ZSo8LACDksav0BNbY0avvv3BckrTu7hsJLgCAiEN4iTD/3+8Oq3fAo0UFafrULVPNLgcAAL8jvESQbXtPaec7TYqOsuif7p0ni4VVcwEAkYfwEiGef6dJ654+KElas2yW5mZPzH4fAEDkI7xEgDdPnNPDv3xLXkP6y5JcPVo22+ySAAAIGMJLmHv3rFNf+Ok+uQe9+kihXd+5bz63iwAAES0kw8t9992ntLQ0ffrTnza7lJDW0N6jz2/Zo66+QZUWpOsHn71ZMdEh+V8pAAB+E5K/dI888oh++tOfml1GSOsf9OoLP9mn1i635mYn60cVJYqPjTa7LAAAAi4kw8vSpUuVnJxsdhkh7T9ertPR5i5lTIrTTx4qlS2B9VwAABPDmMPLrl27tGLFCuXk5MhisWj79u2XHONwOFRQUKD4+HgtXrxYe/bs8UetuOBYc5ccL9VJkv7vPTfJnhJvckUAAATPmMOLy+VSUVGRHA7HZd/ftm2bqqqq9Pjjj2v//v0qKirS8uXL1dLS4jumuLhY8+bNu+TR2Nh4/d9kgvB4DX39129rwGOo7Ea7PrFgitklAQAQVDFj/YPy8nKVl5df8f2NGzdq9erVWrVqlSRp8+bN2rFjh7Zs2aK1a9dKkmpra6+v2stwu91yu92+506n02+fHYq2/ul91TZ0KNkao2+zEB0AYALya89Lf3+/ampqVFZWdvEEUVEqKyvT7t27/Xkqnw0bNshms/keeXl5ATlPKGho79G//uGopKF9i7Jt3C4CAEw8fg0vbW1t8ng8stvtI1632+1qamoa9eeUlZXpM5/5jJ577jnl5uZeNfisW7dOnZ2dvkdDQ8N11x/KDMPQuqcPqnfAo1tnpOuvFkVuSAMA4GrGfNsoGF544YVRH2u1WmW1WuVwOORwOOTxeAJYmXmeqjmt1+raZI2J0nfvX6CoKG4XAQAmJr+OvGRmZio6OlrNzc0jXm9ublZ2drY/T3WJyspKHT58WHv37g3oeczQ4uzTt589LEmq+sgNKsicZHJFAACYx6/hJS4uTgsXLlR1dbXvNa/Xq+rqai1ZssSfp7qEw+FQYWGhFi1aFNDzBJthGPrH7Yfk7BvUglyb/uaO6WaXBACAqcZ826i7u1t1dXW+5/X19aqtrVV6erry8/NVVVWliooKlZSUqLS0VJs2bZLL5fLNPgqUyspKVVZWyul0ymazBfRcwbTj4Fk9f7hZMVEW/fOnFrD8PwBgwhtzeNm3b5+WLVvme15VVSVJqqio0NatW7Vy5Uq1trZq/fr1ampqUnFxsXbu3HlJEy+urd3Vr8d/844k6cvLZunGKSkmVwQAgPkshmEYZhfhDx9s2D127Jg6OzuVkhLeP/aP/uotba9t1A32JD378J2Ki2HUBQAQmYbvnIzm9ztiwsuwsXz5UPbikWY9tHWfoizS01++XcV5qWaXBABAwIzl95t/lQ9BZzp69Y3/PShJ+ps7phNcAAD4gIgJL5Ey26izd0CrfrxHrV1uzbEnq+ojc8wuCQCAkMJtoxDSP+hVxZY92n3inOwpVj3z5duVk5pgdlkAAAQct43CkGEY+sb/vq3dJ85pUly0tjy4iOACAMBlEF5CxMY/HtMzb51RdJRF//HXC3VTTuSsVQMAgD9FTHgJ556X599p0g9eHFr47zv3zdOHbphsckUAAISuiAkv4by30dP7z0iSKpZM08pF+SZXAwBAaIuY8BKuPF5Df3qvTZL0yZunmlwNAAChj/BisrdPd8jZN6iU+BgtmEqfCwAA10J4Mdmrx4dGXW6bmcmmiwAAjELE/FqGa8PuaxfCy503ZJpcCQAA4SFiwks4Nux2uwe1/9R5SdKds5hhBADAaERMeAlHb7x3ToNeQ9MyEpWfkWh2OQAAhAXCi4leqxu6ZXTHLG4ZAQAwWoQXE+063ipJunM2t4wAABgtwotJznT06kSrS1EWacnMDLPLAQAgbERMeAm32UavXRh1KcpLlS0h1uRqAAAIHxETXsJtttHw+i7cMgIAYGwiJryEE6/X0Ot1w+GFZl0AAMaC8GKCdxqdOt8zoCRrjIrzUs0uBwCAsEJ4McGrdUP9LrfOyFAsWwIAADAm/HKa4NVjQ7eM/oItAQAAGDPCS5Cd6ehVzcmhLQFYnA4AgLEjvASR12voa08dUL/Hq0UFaZqeOcnskgAACDsRE17CYZ2Xn715Un9675ziY6P0L58uksViMbskAADCjsUwDMPsIvzJ6XTKZrOps7NTKSkpZpfjU9/m0t3ff1W9Ax59656bVHFbgdklAQAQMsby+x0xIy+hzOM19NWnDqh3wKPbZmboc7dOM7skAADCFuElCP7r1ROqOXleSdYY/cunFygqittFAABcL8JLgB1r7tL3nj8mSVr/iULlpiWaXBEAAOGN8BJg33/huPo9Xi2bM1mfKck1uxwAAMIe4SWAut2DeuHdZknSVz46h9lFAAD4AeElgF443Cz3oFczMifpppzQmfkEAEA4C7nw0tDQoKVLl6qwsFALFizQU089ZXZJ1+13BxolSZ8oymHUBQAAP4kxu4A/FxMTo02bNqm4uFhNTU1auHCh7r77bk2aFF6r0Xb09GvX8aENGFcsmGJyNQAARI6QCy9TpkzRlClDP/bZ2dnKzMxUe3t72IWXP7zTpAGPobnZyZptTza7HAAAIsaYbxvt2rVLK1asUE7O0K2Q7du3X3KMw+FQQUGB4uPjtXjxYu3Zs+e6iqupqZHH41FeXt51/b2ZfnvhltGKohyTKwEAILKMOby4XC4VFRXJ4XBc9v1t27apqqpKjz/+uPbv36+ioiItX75cLS0tvmOKi4s1b968Sx6NjY2+Y9rb2/X5z39eP/zhD6/ja5mrpatPu987J0lasYDwAgCAP435tlF5ebnKy8uv+P7GjRu1evVqrVq1SpK0efNm7dixQ1u2bNHatWslSbW1tVc9h9vt1r333qu1a9fqtttuu+axbrfb99zpdI7ymwTO7w82yWtIRXmpys9gUToAAPzJr7ON+vv7VVNTo7KysosniIpSWVmZdu/eParPMAxDDz74oD784Q/rc5/73DWP37Bhg2w2m+8RCreYhmcZ0agLAID/+TW8tLW1yePxyG63j3jdbrerqalpVJ/x+uuva9u2bdq+fbuKi4tVXFysgwcPXvH4devWqbOz0/doaGgY13cYrzMdvdp38rwsFukT3DICAMDvQm620R133CGv1zvq461Wq6xWqxwOhxwOhzweTwCru7Ydbw+NupQWpCvbFm9qLQAARCK/jrxkZmYqOjpazc3NI15vbm5Wdna2P091icrKSh0+fFh79+4N6Hmu5XcHzkpilhEAAIHi1/ASFxenhQsXqrq62vea1+tVdXW1lixZ4s9ThaTT53t08EynoqMsKp8X2LAGAMBENebbRt3d3aqrq/M9r6+vV21trdLT05Wfn6+qqipVVFSopKREpaWl2rRpk1wul2/2UaCEwm2jo01dkqQb7MnKSLKaVgcAAJFszOFl3759WrZsme95VVWVJKmiokJbt27VypUr1draqvXr16upqUnFxcXauXPnJU28/lZZWanKyko5nU7ZbLaAnutK3mvtliTNykoy5fwAAEwEYw4vS5culWEYVz1mzZo1WrNmzXUXdT1CYeTlvRaXJGlGZnhtZQAAQDgJuV2lr1coNOwOj7zMZOQFAICAiZjwEgp84WUyIy8AAARKxIQXh8OhwsJCLVq0yJTzt7v6db5nQJI0I5ORFwAAAiViwovZt41OXBh1mZqaoIS4aFNqAABgIoiY8GI2+l0AAAgOwoufvNc6NNOIfhcAAAIrYsKL2T0v77UMN+sy8gIAQCBFTHgxu+dl+LbRDEZeAAAIqIgJL2ZyD3p0qr1HkjSLkRcAAAKK8OIHJ8/1yGtIydYYTU5mTyMAAAKJ8OIHw9OkZ2QlyWKxmFwNAACRLWLCi5kNu8w0AgAgeCImvJjZsMtMIwAAgidiwouZLu5pRHgBACDQCC/jZBgGt40AAAgiwss4tXS51e0eVHSURfkZiWaXAwBAxCO8jNNwv0t+eqKsMWzICABAoEVMeDFrttHFfhduGQEAEAwRE17Mmm10sd+FZl0AAIIhYsKLWZhpBABAcBFexunE8MhLFreNAAAIBsLLOPT0D+pMR68kaUYmIy8AAAQD4WUchkdd0ifFKW1SnMnVAAAwMRBexoGZRgAABB/hZRyYaQQAQPBFTHgxY52XE8w0AgAg6CImvJixzstws25eOtsCAAAQLBETXszQ4nRLkrJSrCZXAgDAxEF4uU6GYai1eyi8TE4ivAAAECyEl+vk7B1U/6BXkjQ5mfACAECwEF6uU2t3nyQpJT5G8bHsJg0AQLAQXq7TxX6XeJMrAQBgYiG8XCf6XQAAMEfIhZeOjg6VlJSouLhY8+bN049+9COzS7osZhoBAGCOGLML+HPJycnatWuXEhMT5XK5NG/ePN1///3KyMgwu7QRWrqGel6yaNYFACCoQm7kJTo6WomJQ4u+ud1uGYYhwzBMrupSrV0XbhsRXgAACKoxh5ddu3ZpxYoVysnJkcVi0fbt2y85xuFwqKCgQPHx8Vq8eLH27NkzpnN0dHSoqKhIubm5+trXvqbMzMyxlhlwLRfCS1YyDbsAAATTmMOLy+VSUVGRHA7HZd/ftm2bqqqq9Pjjj2v//v0qKirS8uXL1dLS4jtmuJ/lzx+NjY2SpNTUVB04cED19fX6xS9+oebm5uv8eoHDyAsAAOYYc89LeXm5ysvLr/j+xo0btXr1aq1atUqStHnzZu3YsUNbtmzR2rVrJUm1tbWjOpfdbldRUZFeffVVffrTn77sMW63W2632/fc6XSO8puMz8WRF8ILAADB5Neel/7+ftXU1KisrOziCaKiVFZWpt27d4/qM5qbm9XV1SVJ6uzs1K5duzRnzpwrHr9hwwbZbDbfIy8vb3xfYhTcgx519g5IYuQFAIBg82t4aWtrk8fjkd1uH/G63W5XU1PTqD7j5MmTuvPOO1VUVKQ777xTDz/8sObPn3/F49etW6fOzk7fo6GhYVzfYTSGbxnFxUTJlhAb8PMBAICLQm6qdGlp6ahvK0mS1WqV1WqVw+GQw+GQx+MJXHEX+PpdkqyyWCwBPx8AALjIryMvmZmZio6OvqTBtrm5WdnZ2f481SUqKyt1+PBh7d27N6DnkS72u3DLCACA4PNreImLi9PChQtVXV3te83r9aq6ulpLlizx56lMRbMuAADmGfNto+7ubtXV1fme19fXq7a2Vunp6crPz1dVVZUqKipUUlKi0tJSbdq0SS6Xyzf7KFBMuW1EeAEAIOjGHF727dunZcuW+Z5XVVVJkioqKrR161atXLlSra2tWr9+vZqamlRcXKydO3de0sTrb5WVlaqsrJTT6ZTNZgvouVp9WwOwQB0AAME25vCydOnSay7Xv2bNGq1Zs+a6i7oejLwAADAxhNzeRtfLjIZdel4AAAi+iAkvwTQ88pKVQngBACDYCC9j5PUa3DYCAMBEERNeHA6HCgsLtWjRooCep6N3QIPeoZ6fzCTCCwAAwRYx4SVYPS8tF2YapU+KU2x0xFw+AADCBr++Y9TipFkXAAAzRUx4CdZtI/pdAAAwV8SEl+DdNiK8AABgpogJL8HimybN6roAAJiC8DJGww27jLwAAGAOwssYtbK6LgAApoqY8ELDLgAAE0PEhJdgNewy8gIAgLkiJrwEQ2+/R13uQUmMvAAAYBbCyxgMN+smxEYryRpjcjUAAExMhJcx+OBu0haLxeRqAACYmAgvY+BboI4NGQEAME3EhJdgzDb64MgLAAAwR8SEl2DMNvItUMfICwAApomY8BIMF0de2BoAAACzEF7GgJ4XAADMR3gZgxbnhfBCzwsAAKYhvIxBazer6wIAYDbCyyh5vIbOdbOvEQAAZiO8jNI5l1teQ4qySBmTCC8AAJglYsJLoNd5Ge53yUiyKjqK1XUBADBLxISXQK/zQr8LAAChgd0FRyk/PVGPls1WakKs2aUAADChEV5GaebkJD1adoPZZQAAMOFFzG0jAAAwMRBeAABAWCG8AACAsEJ4AQAAYSVkw0tPT4+mTZumr371q2aXAgAAQkjIhpcnnnhCt956q9llAACAEBOS4eX48eM6cuSIysvLzS4FAACEmDGHl127dmnFihXKycmRxWLR9u3bLznG4XCooKBA8fHxWrx4sfbs2TOmc3z1q1/Vhg0bxloaAACYAMYcXlwul4qKiuRwOC77/rZt21RVVaXHH39c+/fvV1FRkZYvX66WlhbfMcXFxZo3b94lj8bGRv3mN7/RDTfcoBtuYEE4AABwKYthGMZ1/7HFomeeeUb33nuv77XFixdr0aJF+vd//3dJktfrVV5enh5++GGtXbv2mp+5bt06/exnP1N0dLS6u7s1MDCgr3zlK1q/fv1lj3e73XK73b7nTqdTeXl56uzsVEpKyvV+NQAAEEROp1M2m21Uv99+7Xnp7+9XTU2NysrKLp4gKkplZWXavXv3qD5jw4YNamho0Pvvv69//dd/1erVq68YXIaPt9lsvkdeXt64vwcAAAhdfg0vbW1t8ng8stvtI1632+1qamry56l81q1bp87OTt+joaEhIOcBAAChIaQ3ZnzwwQeveYzVapXVapXD4ZDD4ZDH4wl8YQAAwDR+DS+ZmZmKjo5Wc3PziNebm5uVnZ3tz1NdorKyUpWVlers7FRqaqqcTmdAzwcAAPxn+Hd7NK24fg0vcXFxWrhwoaqrq31NvF6vV9XV1VqzZo0/T3VFXV1dkkTvCwAAYairq0s2m+2qx4w5vHR3d6uurs73vL6+XrW1tUpPT1d+fr6qqqpUUVGhkpISlZaWatOmTXK5XFq1atXYv8F1yMnJUUNDg5KTk2WxWPz62cMzmRoaGpjJFGBc6+DhWgcP1zp4uNbB469rbRiGurq6lJOTc81jxxxe9u3bp2XLlvmeV1VVSZIqKiq0detWrVy5Uq2trVq/fr2amppUXFysnTt3XtLEGyhRUVHKzc0N6DlSUlL4f4Yg4VoHD9c6eLjWwcO1Dh5/XOtrjbgMG3N4Wbp06TXvR61ZsyZot4kAAMDEEpJ7GwEAAFwJ4WUMrFarHn/8cVmtVrNLiXhc6+DhWgcP1zp4uNbBY8a1Htf2AAAAAMHGyAsAAAgrhBcAABBWCC8AACCsEF4AAEBYIbyMksPhUEFBgeLj47V48WLt2bPH7JLC3oYNG7Ro0SIlJycrKytL9957r44ePTrimL6+PlVWViojI0NJSUn61Kc+dcneWRi77373u7JYLHr00Ud9r3Gt/efMmTP667/+a2VkZCghIUHz58/Xvn37fO8bhqH169drypQpSkhIUFlZmY4fP25ixeHJ4/Hoscce0/Tp05WQkKCZM2fqn/7pn0asRca1vn67du3SihUrlJOTI4vFou3bt494fzTXtr29XQ888IBSUlKUmpqqv/mbv1F3d/f4izNwTb/61a+MuLg4Y8uWLcY777xjrF692khNTTWam5vNLi2sLV++3Pjxj39sHDp0yKitrTXuvvtuIz8/3+ju7vYd86UvfcnIy8szqqurjX379hm33nqrcdttt5lYdfjbs2ePUVBQYCxYsMB45JFHfK9zrf2jvb3dmDZtmvHggw8ab775pnHixAnjD3/4g1FXV+c75rvf/a5hs9mM7du3GwcOHDDuueceY/r06UZvb6+JlYefJ554wsjIyDCeffZZo76+3njqqaeMpKQk4/vf/77vGK719XvuueeMb37zm8bTTz9tSDKeeeaZEe+P5tp+7GMfM4qKiow33njDePXVV41Zs2YZn/3sZ8ddG+FlFEpLS43Kykrfc4/HY+Tk5BgbNmwwsarI09LSYkgyXnnlFcMwDKOjo8OIjY01nnrqKd8x7777riHJ2L17t1llhrWuri5j9uzZxh//+EfjQx/6kC+8cK395xvf+IZxxx13XPF9r9drZGdnG08++aTvtY6ODsNqtRq//OUvg1FixPj4xz9uPPTQQyNeu//++40HHnjAMAyutT/9eXgZzbU9fPiwIcnYu3ev75jf//73hsViMc6cOTOuerhtdA39/f2qqalRWVmZ77WoqCiVlZVp9+7dJlYWeTo7OyVJ6enpkqSamhoNDAyMuPZz585Vfn4+1/46VVZW6uMf//iIaypxrf3pt7/9rUpKSvSZz3xGWVlZuvnmm/WjH/3I9359fb2amppGXGubzabFixdzrcfotttuU3V1tY4dOyZJOnDggF577TWVl5dL4loH0miu7e7du5WamqqSkhLfMWVlZYqKitKbb745rvOPeW+jiaatrU0ej+eSjSXtdruOHDliUlWRx+v16tFHH9Xtt9+uefPmSZKampoUFxen1NTUEcfa7XY1NTWZUGV4+9WvfqX9+/dr7969l7zHtfafEydO6D//8z9VVVWlf/iHf9DevXv1d3/3d4qLi1NFRYXvel7unylc67FZu3atnE6n5s6dq+joaHk8Hj3xxBN64IEHJIlrHUCjubZNTU3Kysoa8X5MTIzS09PHff0JLwgJlZWVOnTokF577TWzS4lIDQ0NeuSRR/THP/5R8fHxZpcT0bxer0pKSvSd73xHknTzzTfr0KFD2rx5syoqKkyuLrL8z//8j37+85/rF7/4hW666SbV1tbq0UcfVU5ODtc6wnHb6BoyMzMVHR19yayL5uZmZWdnm1RVZFmzZo2effZZvfTSS8rNzfW9np2drf7+fnV0dIw4nms/djU1NWppadEtt9yimJgYxcTE6JVXXtG//du/KSYmRna7nWvtJ1OmTFFhYeGI12688UadOnVKknzXk3+mjN/XvvY1rV27Vn/1V3+l+fPn63Of+5z+/u//Xhs2bJDEtQ6k0Vzb7OxstbS0jHh/cHBQ7e3t477+hJdriIuL08KFC1VdXe17zev1qrq6WkuWLDGxsvBnGIbWrFmjZ555Ri+++KKmT58+4v2FCxcqNjZ2xLU/evSoTp06xbUfo7vuuksHDx5UbW2t71FSUqIHHnjA939zrf3j9ttvv2TK/7FjxzRt2jRJ0vTp05WdnT3iWjudTr355ptc6zHq6elRVNTIn7Ho6Gh5vV5JXOtAGs21XbJkiTo6OlRTU+M75sUXX5TX69XixYvHV8C42n0niF/96leG1Wo1tm7dahw+fNj44he/aKSmphpNTU1mlxbW/vZv/9aw2WzGyy+/bJw9e9b36Onp8R3zpS99ycjPzzdefPFFY9++fcaSJUuMJUuWmFh15PjgbCPD4Fr7y549e4yYmBjjiSeeMI4fP278/Oc/NxITE42f/exnvmO++93vGqmpqcZvfvMb4+233zY++clPMn33OlRUVBhTp071TZV++umnjczMTOPrX/+67xiu9fXr6uoy3nrrLeOtt94yJBkbN2403nrrLePkyZOGYYzu2n7sYx8zbr75ZuPNN980XnvtNWP27NlMlQ6mH/zgB0Z+fr4RFxdnlJaWGm+88YbZJYU9SZd9/PjHP/Yd09vba3z5y1820tLSjMTEROO+++4zzp49a17REeTPwwvX2n9+97vfGfPmzTOsVqsxd+5c44c//OGI971er/HYY48ZdrvdsFqtxl133WUcPXrUpGrDl9PpNB555BEjPz/fiI+PN2bMmGF885vfNNxut+8YrvX1e+mlly77z+iKigrDMEZ3bc+dO2d89rOfNZKSkoyUlBRj1apVRldX17hrsxjGB5YiBAAACHH0vAAAgLBCeAEAAGGF8AIAAMIK4QUAAIQVwgsAAAgrhBcAABBWCC8AACCsEF4AAEBYIbwAAICwQngBAABhhfACAADCCuEFAACElf8f11hueST/g+YAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# dummy gram matrix\n", + "\n", + "size = 100\n", + "rand_array = np.random.rand(size, size)\n", + "\n", + "G = np.dot(rand_array, rand_array.T)\n", + "\n", + "vals, vects = np.linalg.eigh(G)\n", + "\n", + "\n", + "plt.plot(vals)\n", + "plt.yscale('log')" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([[ 0, 1, 2, 3, 4],\n", + " [ 5, 6, 7, 8, 9],\n", + " [10, 11, 12, 13, 14]]),\n", + " array([[10, 11, 12, 13, 14],\n", + " [ 5, 6, 7, 8, 9],\n", + " [ 0, 1, 2, 3, 4]]))" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = np.arange(15).reshape(3, 5)\n", + "a, a[::-1]" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([-0.70710678, -0.70710678, -0. ]),\n", + " array([0.70710678, 0.70710678, 0. ]))" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Create a 2x2 hermitian matrix of known eigenvalues and eigenvectors\n", + "\n", + "# eigenvalues\n", + "l1 = 1\n", + "l2 = 2\n", + "l3 = 3\n", + "\n", + "# eigenvectors\n", + "v1 = np.array([1, 1, 0]) / np.sqrt(2)\n", + "v2 = np.array([1, -1, 0]) / np.sqrt(2)\n", + "v3 = np.array([0, 0, 1])\n", + "\n", + "# create the matrix\n", + "M = l1 * np.outer(v1, v1.conj()) + l2 * np.outer(v2, v2.conj()) + l3 * np.outer(v3, v3.conj())\n", + "\n", + "# check if it is hermitian\n", + "assert np.allclose(M, M.T.conj())\n", + "\n", + "# check if the eigenvectors are correct\n", + "vals, vects = np.linalg.eigh(M)\n", + "vals, vects\n", + "assert np.allclose(vals, [l1, l2, l3])\n", + "assert np.allclose(-vects[:, 0], v1)\n", + "assert np.allclose(-vects[:, 1], v2)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eigvals = np.array([1, 2])\n", + "# Create a random 2x2 matrix\n", + "A = np.random.randn(2, 2)\n", + "# Construct the matrix with the specified eigenvalues\n", + "B = np.dot(np.dot(A, np.diag(eigvals)), np.linalg.inv(A))\n", + "# Compute the eigenvalues of the constructed matrix\n", + "np.linalg.eigvals(B)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eigvals = np.array([1, 2])\n", + "# Create a random 2x2 matrix\n", + "A = np.random.randn(2, 2)\n", + "# Construct the matrix with the specified eigenvalues\n", + "B = np.dot(np.dot(A, np.diag(eigvals)), np.linalg.inv(A))\n", + "# Compute the eigenvalues of the constructed matrix\n", + "np.linalg.eigvals(B)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "jax_gpu", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}