diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..ecf211f --- /dev/null +++ b/.coveragerc @@ -0,0 +1,28 @@ +# .coveragerc to control coverage.py +[run] +branch = True +source = keras_explainable +# omit = bad_file.py + +[paths] +source = + src/ + */site-packages/ + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..3dff310 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,39 @@ +name: Pages +on: + push: + tags: + - '*' +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: actions/checkout@master + with: + fetch-depth: 0 + - name: Cache Dependencies + uses: actions/cache@v2 + id: cache + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Dependencies + run: | + echo "Installing dependencies and caching them." + python -m pip install --upgrade pip + pip install numpy pandas matplotlib + pip install https://files.pythonhosted.org/packages/04/ea/49fd026ac36fdd79bf072294b139170aefc118e487ccb39af019946797e9/tensorflow-2.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + pip install . + - name: Build and Commit + uses: sphinx-notes/pages@v2 + with: + requirements_path: ./docs/requirements.txt + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: gh-pages diff --git a/.gitignore b/.gitignore index b6e4761..e9e1e9b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,129 +1,54 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ +# Temporary and binary files +*~ *.py[cod] -*$py.class - -# C extensions *.so +*.cfg +!.isort.cfg +!setup.cfg +*.orig +*.log +*.pot +__pycache__/* +.cache/* +.*.swp +*/.ipynb_checkpoints/* +.DS_Store -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg +# Project files +.ropeproject +.project +.pydevproject +.settings +.idea +.vscode +tags + +# Package files *.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt +*.eggs/ +.installed.cfg +*.egg-info -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ +# Unittest and coverage +htmlcov/* .coverage .coverage.* -.cache -nosetests.xml +.tox +junit*.xml coverage.xml -*.cover -*.py,cover -.hypothesis/ .pytest_cache/ -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py +# Build and docs folder/files +build/* +dist/* +sdist/* +docs/api/* +docs/_rst/* +docs/_build/* +cover/* +MANIFEST -# pyenv +# Per-project virtualenvs +.venv*/ +.conda*/ .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..21b0814 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,23 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF +formats: + - pdf + +python: + version: 3.8 + install: + - requirements: docs/requirements.txt + - {path: ., method: pip} diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000..a9f28dd --- /dev/null +++ b/.style.yapf @@ -0,0 +1,11 @@ +[style] +based_on_style = google +spaces_before_comment = 2 +indent_width = 2 +split_before_logical_operator = true +column_limit = 90 +split_before_named_assigns = true +dedent_closing_brackets = true +indent_dictionary_value = false +continuation_indent_width = 2 +split_before_default_or_named_assigns = true diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..678c4d1 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,5 @@ +============ +Contributors +============ + +* Lucas David diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..2fcfdf3 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,17 @@ +========= +Changelog +========= + +Version 0.1 +=========== + +- Start project with scaffolding and add schematics +- Add CAM explaining method +- Add Grad-CAM explaining method +- Add Grad-CAM++ explaining method +- Add Score-CAM explaining method +- Add gradient backprop saliency explaining method +- Add FullGrad explaining method +- Add `engine`, `filters` and `inspection` modules +- Add :py:mod:`keras_explainable.methods.meta` module, containing + implementations for the ``Smooth`` and ``TTA`` procedures diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..b7e7bb5 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,318 @@ +============ +Contributing +============ + +Welcome to ``keras-explainable`` contributor's guide. + +This document focuses on getting any potential contributor familiarized +with the development processes, but `other kinds of contributions`_ are also +appreciated. + +If you are new to using git_ or have never collaborated in a project previously, +please have a look at `contribution-guide.org`_. Other resources are also +listed in the excellent `guide created by FreeCodeCamp`_ [#contrib1]_. + +Please notice, all users and contributors are expected to be **open, +considerate, reasonable, and respectful**. When in doubt, `Python Software +Foundation's Code of Conduct`_ is a good reference in terms of behavior +guidelines. + +Issue Reports +============= + +If you experience bugs or general issues with ``keras-explainable``, please have a look +on the `issue tracker`_. If you don't see anything useful there, please feel +free to fire an issue report. + +.. tip:: + Please don't forget to include the closed issues in your search. + Sometimes a solution was already reported, and the problem is considered + **solved**. + +New issue reports should include information about your programming environment +(e.g., operating system, Python version) and steps to reproduce the problem. +Please try also to simplify the reproduction steps to a very minimal example +that still illustrates the problem you are facing. By removing other factors, +you help us to identify the root cause of the issue. + +Documentation Improvements +========================== + +You can help improve ``keras-explainable`` docs by making them more readable and coherent, or +by adding missing information and correcting mistakes. + +``keras-explainable`` documentation uses Sphinx_ as its main documentation compiler. +This means that the docs are kept in the same repository as the project code, and +that any documentation update is done in the same way was a code contribution. + +.. todo:: Don't forget to mention which markup language you are using. + + e.g., reStructuredText_ or CommonMark_ with MyST_ extensions. + +.. todo:: If your project is hosted on GitHub, you can also mention the following tip: + + .. tip:: + Please notice that the `GitHub web interface`_ provides a quick way of + propose changes in ``keras-explainable``'s files. While this mechanism can + be tricky for normal code contributions, it works perfectly fine for + contributing to the docs, and can be quite handy. + + If you are interested in trying this method out, please navigate to + the ``docs`` folder in the source repository_, find which file you + would like to propose changes and click in the little pencil icon at the + top, to open `GitHub's code editor`_. Once you finish editing the file, + please write a message in the form at the bottom of the page describing + which changes have you made and what are the motivations behind them and + submit your proposal. + +When working on documentation changes in your local machine, you can +compile them using |tox|_:: + + tox -e docs + +and use Python's built-in web server for a preview in your web browser +(``http://localhost:8000``):: + + python3 -m http.server --directory 'docs/_build/html' + +Code Contributions +================== + +.. todo:: Please include a reference or explanation about the internals of the project. + + An architecture description, design principles or at least a summary of the + main concepts will make it easy for potential contributors to get started + quickly. + +Submit an issue +--------------- + +Before you work on any non-trivial code contribution it's best to first create +a report in the `issue tracker`_ to start a discussion on the subject. +This often provides additional considerations and avoids unnecessary work. + +Create an environment +--------------------- + +Before you start coding, we recommend creating an isolated `virtual +environment`_ to avoid any problems with your installed Python packages. +This can easily be done via either |virtualenv|_:: + + virtualenv + source /bin/activate + +or Miniconda_:: + + conda create -n keras-explainable python=3 six virtualenv pytest pytest-cov + conda activate keras-explainable + +Clone the repository +-------------------- + +#. Create an user account on |the repository service| if you do not already have one. +#. Fork the project repository_: click on the *Fork* button near the top of the + page. This creates a copy of the code under your account on |the repository service|. +#. Clone this copy to your local disk:: + + git clone git@github.com:YourLogin/keras-explainable.git + cd keras-explainable + +#. You should run:: + + pip install -U pip setuptools -e . + + to be able to import the package under development in the Python REPL. + + .. todo:: if you are not using pre-commit, please remove the following item: + +#. Install |pre-commit|_:: + + pip install pre-commit + pre-commit install + + ``keras-explainable`` comes with a lot of hooks configured to automatically help the + developer to check the code being written. + +Implement your changes +---------------------- + +#. Create a branch to hold your changes:: + + git checkout -b my-feature + + and start making changes. Never work on the main branch! + +#. Start your work on this branch. Don't forget to add docstrings_ to new + functions, modules and classes, especially if they are part of public APIs. + +#. Add yourself to the list of contributors in ``AUTHORS.rst``. + +#. When you’re done editing, do:: + + git add + git commit + + to record your changes in git_. + + .. todo:: if you are not using pre-commit, please remove the following item: + + Please make sure to see the validation messages from |pre-commit|_ and fix + any eventual issues. + This should automatically use flake8_/black_ to check/fix the code style + in a way that is compatible with the project. + + .. important:: Don't forget to add unit tests and documentation in case your + contribution adds an additional feature and is not just a bugfix. + + Moreover, writing a `descriptive commit message`_ is highly recommended. + In case of doubt, you can check the commit history with:: + + git log --graph --decorate --pretty=oneline --abbrev-commit --all + + to look for recurring communication patterns. + +#. Please check that your changes don't break any unit tests with:: + + tox + + (after having installed |tox|_ with ``pip install tox`` or ``pipx``). + + You can also use |tox|_ to run several other pre-configured tasks in the + repository. Try ``tox -av`` to see a list of the available checks. + +Submit your contribution +------------------------ + +#. If everything works fine, push your local branch to |the repository service| with:: + + git push -u origin my-feature + +#. Go to the web page of your fork and click |contribute button| + to send your changes for review. + + .. todo:: if you are using GitHub, you can uncomment the following paragraph + + Find more detailed information in `creating a PR`_. You might also want to open + the PR as a draft first and mark it as ready for review after the feedbacks + from the continuous integration (CI) system or any required fixes. + +Troubleshooting +--------------- + +The following tips can be used when facing problems to build or test the +package: + +#. Make sure to fetch all the tags from the upstream repository_. + The command ``git describe --abbrev=0 --tags`` should return the version you + are expecting. If you are trying to run CI scripts in a fork repository, + make sure to push all the tags. + You can also try to remove all the egg files or the complete egg folder, i.e., + ``.eggs``, as well as the ``*.egg-info`` folders in the ``src`` folder or + potentially in the root of your project. + +#. Sometimes |tox|_ misses out when new dependencies are added, especially to + ``setup.cfg`` and ``docs/requirements.txt``. If you find any problems with + missing dependencies when running a command with |tox|_, try to recreate the + ``tox`` environment using the ``-r`` flag. For example, instead of:: + + tox -e docs + + Try running:: + + tox -r -e docs + +#. Make sure to have a reliable |tox|_ installation that uses the correct + Python version (e.g., 3.7+). When in doubt you can run:: + + tox --version + # OR + which tox + + If you have trouble and are seeing weird errors upon running |tox|_, you can + also try to create a dedicated `virtual environment`_ with a |tox|_ binary + freshly installed. For example:: + + virtualenv .venv + source .venv/bin/activate + .venv/bin/pip install tox + .venv/bin/tox -e all + +#. `Pytest can drop you`_ in an interactive session in the case an error occurs. + In order to do that you need to pass a ``--pdb`` option (for example by + running ``tox -- -k --pdb``). + You can also setup breakpoints manually instead of using the ``--pdb`` option. + +Maintainer tasks +================ + +Releases +-------- + +.. todo:: This section assumes you are using PyPI to publicly release your package. + + If instead you are using a different/private package index, please update + the instructions accordingly. + +If you are part of the group of maintainers and have correct user permissions +on PyPI_, the following steps can be used to release a new version for +``keras-explainable``: + +#. Make sure all unit tests are successful. +#. Tag the current commit on the main branch with a release tag, e.g., ``v1.2.3``. +#. Push the new tag to the upstream repository_, e.g., ``git push upstream v1.2.3`` +#. Clean up the ``dist`` and ``build`` folders with ``tox -e clean`` + (or ``rm -rf dist build``) + to avoid confusion with old builds and Sphinx docs. +#. Run ``tox -e build`` and check that the files in ``dist`` have + the correct version (no ``.dirty`` or git_ hash) according to the git_ tag. + Also check the sizes of the distributions, if they are too big (e.g., > + 500KB), unwanted clutter may have been accidentally included. +#. Run ``tox -e publish -- --repository pypi`` and check that everything was + uploaded to PyPI_ correctly. + +.. [#contrib1] Even though, these resources focus on open source projects and + communities, the general ideas behind collaborating with other developers + to collectively create software are general and can be applied to all sorts + of environments, including private companies and proprietary code bases. + +.. <-- strart --> +.. todo:: Please review and change the following definitions: + +.. |the repository service| replace:: GitHub +.. |contribute button| replace:: "Create pull request" + +.. _repository: https://github.com/lucasdavid/keras-explainable +.. _issue tracker: https://github.com/lucasdavid/keras-explainable/issues +.. <-- end --> + +.. |virtualenv| replace:: ``virtualenv`` +.. |pre-commit| replace:: ``pre-commit`` +.. |tox| replace:: ``tox`` + +.. _black: https://pypi.org/project/black/ +.. _CommonMark: https://commonmark.org/ +.. _contribution-guide.org: https://www.contribution-guide.org/ +.. _creating a PR: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request +.. _descriptive commit message: https://chris.beams.io/posts/git-commit +.. _docstrings: https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html +.. _first-contributions tutorial: https://github.com/firstcontributions/first-contributions +.. _flake8: https://flake8.pycqa.org/en/stable/ +.. _git: https://git-scm.com +.. _GitHub's fork and pull request workflow: https://guides.github.com/activities/forking/ +.. _guide created by FreeCodeCamp: https://github.com/FreeCodeCamp/how-to-contribute-to-open-source +.. _Miniconda: https://docs.conda.io/en/latest/miniconda.html +.. _MyST: https://myst-parser.readthedocs.io/en/latest/syntax/syntax.html +.. _other kinds of contributions: https://opensource.guide/how-to-contribute +.. _pre-commit: https://pre-commit.com/ +.. _PyPI: https://pypi.org/ +.. _PyScaffold's contributor's guide: https://pyscaffold.org/en/stable/contributing.html +.. _Pytest can drop you: https://docs.pytest.org/en/stable/how-to/failures.html#using-python-library-pdb-with-pytest +.. _Python Software Foundation's Code of Conduct: https://www.python.org/psf/conduct/ +.. _reStructuredText: https://www.sphinx-doc.org/en/master/usage/restructuredtext/ +.. _Sphinx: https://www.sphinx-doc.org/en/master/ +.. _tox: https://tox.wiki/en/stable/ +.. _virtual environment: https://realpython.com/python-virtual-environments-a-primer/ +.. _virtualenv: https://virtualenv.pypa.io/en/stable/ + +.. _GitHub web interface: https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files +.. _GitHub's code editor: https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files diff --git a/LICENSE b/LICENSE index 261eeb9..15b2886 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,201 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2022 Lucas David + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md deleted file mode 100644 index 004d49f..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# keras-explainable -Efficient explaining AI algorithms for Keras models diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..dad1e5b --- /dev/null +++ b/README.rst @@ -0,0 +1,29 @@ +================= +keras-explainable +================= + +Efficient explaining AI algorithms for Keras models. + +Installation +------------ + +.. code-block:: shell + + pip install tensorflow + pip install git+https://github.com/lucasdavid/keras-explainable.git + +Usage +----- + +This example illustrate how to explain predictions of a Convolutional Neural +Network (CNN) using Grad-CAM. This can be easily achieved with the following +example: + +.. code-block:: python + + import keras_explainable as ke + + model = tf.keras.applications.ResNet50V2(...) + model = ke.inspection.expose(model) + + scores, cams = ke.gradcam(model, x, y, batch_size=32) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..31655dd --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,29 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build +AUTODOCDIR = api + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $?), 1) +$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from https://sphinx-doc.org/") +endif + +.PHONY: help clean Makefile + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + rm -rf $(BUILDDIR)/* $(AUTODOCDIR) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/.gitignore b/docs/_static/.gitignore new file mode 100644 index 0000000..3c96363 --- /dev/null +++ b/docs/_static/.gitignore @@ -0,0 +1 @@ +# Empty directory diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 0000000..2d22768 --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1,77 @@ +.jupyter_container { + background-color: transparent !important; + border: none !important; + margin: .85rem 0 !important; + + -webkit-box-shadow: none !important; + -moz-box-shadow: none !important; + box-shadow: none !important; +} + +.code_cell { + border: none !important; + background-color: transparent !important; + border-radius: 0 !important; +} + +.highlight { +} + +.highlight>pre { + background-color: #f7f7f7!important; + + padding: 1.25em !important; + margin: .85rem 0 !important; + + border: none !important; + border-radius: 0 !important; + + -webkit-box-shadow: none !important; + -moz-box-shadow: none !important; + box-shadow: none !important; +} + +.dataframe { + border: none !important; +} + +input[type="text"] { + display: block; + width: 100%; + padding: .375rem .75rem; + font-size: 1rem; + line-height: 1.5; + color: #495057; + background-color: #fff; + background-clip: padding-box; + border: 1px solid #ced4da; + border-radius: .25rem; + transition: border-color .15s ease-in-out, box-shadow .15s ease-in-out; +} + +input[type="text"]:focus { + border-color: #007daf; + box-shadow: 0 0 0 3px rgba(54, 198, 255, .25); +} + +.function>dt, +.method>dt { + overflow-x: auto; +} + +table { + color: #666; + border: #eee 1px solid; + width: 100%; +} + +table th { + border: 0; + padding: 0.2em; +} + +table td { + text-align: right; + border: #efefef 1px solid; + padding: 0.2em; +} diff --git a/docs/_static/images/Dalmatian-2.jpg b/docs/_static/images/Dalmatian-2.jpg new file mode 100644 index 0000000..a98139b Binary files /dev/null and b/docs/_static/images/Dalmatian-2.jpg differ diff --git a/docs/_static/images/ILSVRC2012_val_00000073.JPEG b/docs/_static/images/ILSVRC2012_val_00000073.JPEG new file mode 100644 index 0000000..5608367 Binary files /dev/null and b/docs/_static/images/ILSVRC2012_val_00000073.JPEG differ diff --git a/docs/_static/images/ILSVRC2012_val_00000091.JPEG b/docs/_static/images/ILSVRC2012_val_00000091.JPEG new file mode 100644 index 0000000..1e83e63 Binary files /dev/null and b/docs/_static/images/ILSVRC2012_val_00000091.JPEG differ diff --git a/docs/_static/images/ILSVRC2012_val_00000198.JPEG b/docs/_static/images/ILSVRC2012_val_00000198.JPEG new file mode 100644 index 0000000..658e245 Binary files /dev/null and b/docs/_static/images/ILSVRC2012_val_00000198.JPEG differ diff --git a/docs/_static/images/ILSVRC2012_val_00000476.JPEG b/docs/_static/images/ILSVRC2012_val_00000476.JPEG new file mode 100644 index 0000000..6242d8a Binary files /dev/null and b/docs/_static/images/ILSVRC2012_val_00000476.JPEG differ diff --git a/docs/_static/images/ILSVRC2012_val_00002193.JPEG b/docs/_static/images/ILSVRC2012_val_00002193.JPEG new file mode 100644 index 0000000..d98164a Binary files /dev/null and b/docs/_static/images/ILSVRC2012_val_00002193.JPEG differ diff --git a/docs/_static/images/Images-of-San-Francisco-Garter-Snake.jpg b/docs/_static/images/Images-of-San-Francisco-Garter-Snake.jpg new file mode 100644 index 0000000..a572045 Binary files /dev/null and b/docs/_static/images/Images-of-San-Francisco-Garter-Snake.jpg differ diff --git a/docs/_static/images/_links.txt b/docs/_static/images/_links.txt new file mode 100644 index 0000000..7b8d319 --- /dev/null +++ b/docs/_static/images/_links.txt @@ -0,0 +1,15 @@ +https://raw.githubusercontent.com/haofanwang/Score-CAM/master/images/ILSVRC2012_val_00000073.JPEG +https://raw.githubusercontent.com/haofanwang/Score-CAM/master/images/ILSVRC2012_val_00000091.JPEG +https://raw.githubusercontent.com/haofanwang/Score-CAM/master/images/ILSVRC2012_val_00000198.JPEG +https://raw.githubusercontent.com/haofanwang/Score-CAM/master/images/ILSVRC2012_val_00000476.JPEG +https://raw.githubusercontent.com/haofanwang/Score-CAM/master/images/ILSVRC2012_val_00002193.JPEG +https://raw.githubusercontent.com/keisen/tf-keras-vis/master/docs/examples/images/goldfish.jpg +https://raw.githubusercontent.com/keisen/tf-keras-vis/master/docs/examples/images/bear.jpg +https://raw.githubusercontent.com/keisen/tf-keras-vis/master/docs/examples/images/soldiers.jpg +https://3.bp.blogspot.com/-W__wiaHUjwI/Vt3Grd8df0I/AAAAAAAAA78/7xqUNj8ujtY/s400/image02.png +http://www.aviationexplorer.com/Diecast_Airplanes_Aircraft/delta_Airbus_diecast_airplane.jpg +https://www.petcare.com.au/wp-content/uploads/2017/09/Dalmatian-2.jpg +http://sites.psu.edu/siowfa15/wp-content/uploads/sites/29639/2015/10/dogcat.jpg +https://consciouscat.net/wp-content/uploads/2009/08/multiple-cats-300x225.jpg +https://images2.minutemediacdn.com/image/upload/c_crop,h_843,w_1500,x_0,y_78/f_auto,q_auto,w_1100/v1554995977/shape/mentalfloss/iStock-157312120.jpg +http://www.reptilefact.com/wp-content/uploads/2016/08/Images-of-San-Francisco-Garter-Snake.jpg diff --git a/docs/_static/images/bear.jpg b/docs/_static/images/bear.jpg new file mode 100644 index 0000000..d06a836 Binary files /dev/null and b/docs/_static/images/bear.jpg differ diff --git a/docs/_static/images/delta_Airbus_diecast_airplane.jpg b/docs/_static/images/delta_Airbus_diecast_airplane.jpg new file mode 100644 index 0000000..6a6de09 Binary files /dev/null and b/docs/_static/images/delta_Airbus_diecast_airplane.jpg differ diff --git a/docs/_static/images/dogcat.jpg b/docs/_static/images/dogcat.jpg new file mode 100644 index 0000000..8476886 Binary files /dev/null and b/docs/_static/images/dogcat.jpg differ diff --git a/docs/_static/images/goldfish.jpg b/docs/_static/images/goldfish.jpg new file mode 100644 index 0000000..70d50f4 Binary files /dev/null and b/docs/_static/images/goldfish.jpg differ diff --git a/docs/_static/images/iStock-157312120.webp b/docs/_static/images/iStock-157312120.webp new file mode 100644 index 0000000..0943847 Binary files /dev/null and b/docs/_static/images/iStock-157312120.webp differ diff --git a/docs/_static/images/image02.png b/docs/_static/images/image02.png new file mode 100644 index 0000000..c28a02f Binary files /dev/null and b/docs/_static/images/image02.png differ diff --git a/docs/_static/images/multiple-cats-300x225.jpg b/docs/_static/images/multiple-cats-300x225.jpg new file mode 100644 index 0000000..61b7dcf Binary files /dev/null and b/docs/_static/images/multiple-cats-300x225.jpg differ diff --git a/docs/_static/images/soldiers.jpg b/docs/_static/images/soldiers.jpg new file mode 100644 index 0000000..ad56aa1 Binary files /dev/null and b/docs/_static/images/soldiers.jpg differ diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 0000000..cd8e091 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1,2 @@ +.. _authors: +.. include:: ../AUTHORS.rst diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..871950d --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,2 @@ +.. _changes: +.. include:: ../CHANGELOG.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..985a4d3 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,290 @@ +# This file is execfile()d with the current directory set to its containing dir. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import sys +import shutil + +# -- Path setup -------------------------------------------------------------- + +__location__ = os.path.dirname(__file__) + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.join(__location__, "../src")) + +# -- Run sphinx-apidoc ------------------------------------------------------- +# This hack is necessary since RTD does not issue `sphinx-apidoc` before running +# `sphinx-build -b html . _build/html`. See Issue: +# https://github.com/readthedocs/readthedocs.org/issues/1139 +# DON'T FORGET: Check the box "Install your project inside a virtualenv using +# setup.py install" in the RTD Advanced Settings. +# Additionally it helps us to avoid running apidoc manually + +try: # for Sphinx >= 1.7 + from sphinx.ext import apidoc +except ImportError: + from sphinx import apidoc + +output_dir = os.path.join(__location__, "api") +module_dir = os.path.join(__location__, "../src/keras_explainable") +try: + shutil.rmtree(output_dir) +except FileNotFoundError: + pass + +try: + import sphinx + + cmd_line = f"sphinx-apidoc --implicit-namespaces -f -o {output_dir} {module_dir}" + + args = cmd_line.split(" ") + if tuple(sphinx.__version__.split(".")) >= ("1", "7"): + # This is a rudimentary parse_version to avoid external dependencies + args = args[1:] + + apidoc.main(args) +except Exception as e: + print("Running `sphinx-apidoc` failed!\n{}".format(e)) + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.autosummary", + "sphinx.ext.viewcode", + "sphinx.ext.coverage", + "sphinx.ext.doctest", + "sphinx.ext.ifconfig", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "jupyter_sphinx", +] + +# from sphinx_execute_code import directives + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix of source filenames. +source_suffix = ".rst" + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "keras-explainable" +copyright = "2022, Lucas David" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# version: The short X.Y version. +# release: The full version, including alpha/beta/rc tags. +# If you don’t need the separation provided between version and release, +# just set them both to the same value. +try: + from keras_explainable import __version__ as version +except ImportError: + version = "" + +if not version or version.lower() == "unknown": + version = os.getenv("READTHEDOCS_VERSION", "unknown") # automatically set by RTD + +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".venv"] + +# The reST default role (used for this markup: `text`) to use for all documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "arduino" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If this is True, todo emits a warning for each TODO entries. The default is False. +todo_emit_warnings = True + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_book_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {"sidebar_width": "300px", "page_width": "1200px"} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +html_title = "Keras Explainable" + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = "" + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = "keras-explainable-doc" + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ("letterpaper" or "a4paper"). + # "papersize": "letterpaper", + # The font size ("10pt", "11pt" or "12pt"). + # "pointsize": "10pt", + # Additional stuff for the LaTeX preamble. + # "preamble": "", +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ("index", "user_guide.tex", "keras-explainable Documentation", "Lucas David", "manual") +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = "" + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + +# -- External mapping -------------------------------------------------------- +python_version = ".".join(map(str, sys.version_info[0:2])) +intersphinx_mapping = { + "sphinx": ("https://www.sphinx-doc.org/en/master", None), + "python": ("https://docs.python.org/" + python_version, None), + "matplotlib": ("https://matplotlib.org", None), + "numpy": ("https://numpy.org/doc/stable", None), + "sklearn": ("https://scikit-learn.org/stable", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), + "setuptools": ("https://setuptools.pypa.io/en/stable/", None), + "pyscaffold": ("https://pyscaffold.org/en/stable", None), +} + +print(f"loading configurations for {project} {version} ...", file=sys.stderr) + +def builder_inited(app): + app.add_css_file('css/custom.css') + +def setup(app): + app.connect('builder-inited', builder_inited) diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..e582053 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/explaining.rst b/docs/explaining.rst new file mode 100644 index 0000000..67d2c12 --- /dev/null +++ b/docs/explaining.rst @@ -0,0 +1,113 @@ +============================== +Explaining Model's Predictions +============================== + +This library has the function :py:func:`keras_explainable.explain` as core +component, which is used to execute any AI explaining method and technique. + +Think of it as the :py:meth:`keras.Model#fit` or :py:meth:`keras.Model#predict` +loops of Keras' models, in which the execution graph of the operations +contained in a model is compiled (conditioned to :py:attr:`Model.run_eagerly` +and :py:attr:`Model.jit_compile`) and the explaining maps are computed +according to the method's strategy. + +Just like in :py:meth:`keras.model#predict`, :py:func:`keras_explainable.explain` +allows various types of input data and retrieves the Model's associated +distribute strategy in order to distribute the workload across multiple +GPUs and/or workers. + +.. jupyter-execute:: + :hide-code: + :hide-output: + + import os + import numpy as np + import pandas as pd + import tensorflow as tf + from keras.utils import load_img, img_to_array + + import keras_explainable as ke + + SOURCE_DIRECTORY = '_static/images/' + SAMPLES = 8 + SIZES = (299, 299) + + file_names = os.listdir(SOURCE_DIRECTORY) + image_paths = [os.path.join(SOURCE_DIRECTORY, f) + for f in file_names + if f != '_links.txt'] + + images = np.stack([img_to_array(load_img(ip).resize(SIZES)) for ip in image_paths]) + + print('Images shape =', images.shape[1:]) + print('Images avail =', len(images)) + print('Images used =', SAMPLES) + + images = images[:SAMPLES] + +Firstly, we employ the :py:class:`ResNet101` network pre-trained over the +ImageNet dataset: + +.. jupyter-execute:: + + WEIGHTS = 'imagenet' + + input_tensor = tf.keras.Input(shape=(*SIZES, 3), name='inputs') + + rn101 = tf.keras.applications.ResNet101V2( + input_tensor=input_tensor, + classifier_activation=None, + weights=WEIGHTS + ) + rn101.trainable = False + rn101.compile( + optimizer='sgd', + loss='sparse_categorical_crossentropy', + ) + + prec = tf.keras.applications.resnet_v2.preprocess_input + decode_predictions = tf.keras.applications.resnet_v2.decode_predictions + + print(f'ResNet101 with {WEIGHTS} pre-trained weights loaded.') + print(f"Spatial map sizes: {rn101.get_layer('avg_pool').input.shape}") + +We can feed-foward the samples once and get the predicted classes for each sample. +Besides making sure the model is outputing the expected classes, this step is +required in order to determine the most activating units in the *logits* layer, +which improves performance of the explaining methods. + +.. jupyter-execute:: + + inputs = prec(images.copy()) + logits = rn101.predict(inputs, verbose=0) + + indices = np.argsort(logits, axis=-1)[:, ::-1] + probs = tf.nn.softmax(logits).numpy() + predictions = decode_predictions(probs, top=1) + +Finally, we can simply run all available explaining methods: + +.. jupyter-execute:: + + rn101 = ke.inspection.expose(rn101) + + explaining_units = indices[:, :1] # First most likely class. + + _, c_maps = ke.cam(rn101, inputs, explaining_units) + _, gc_maps = ke.gradcam(rn101, inputs, explaining_units) + _, gcpp_maps = ke.gradcampp(rn101, inputs, explaining_units) + _, sc_maps = ke.scorecam(rn101, inputs, explaining_units) + +Following the original Grad-CAM paper, we only consider the positive contributing regions +in the creation of the CAMs, crunching negatively contributing and non-related regions together: + +.. jupyter-execute:: + + all_maps = (c_maps, gc_maps, gcpp_maps, sc_maps) + images = images.astype(np.uint8).repeat(1 + len(all_maps), axis=0) + + ke.utils.visualize( + images=images, + overlay=sum(zip([None] * len(images), *all_maps), ()), + cols=1 + len(all_maps), + ) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..95cf433 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,25 @@ +.. include:: ./readme.rst + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + Overview + Explaining + Methods + Contributions & Help + License + Authors + Changelog + Module Reference + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + +.. _Sphinx: https://www.sphinx-doc.org/ diff --git a/docs/license.rst b/docs/license.rst new file mode 100644 index 0000000..e647e18 --- /dev/null +++ b/docs/license.rst @@ -0,0 +1,7 @@ +.. _license: + +======= +License +======= + +.. include:: ../LICENSE diff --git a/docs/methods/cams/gradcam.rst b/docs/methods/cams/gradcam.rst new file mode 100644 index 0000000..abc4f97 --- /dev/null +++ b/docs/methods/cams/gradcam.rst @@ -0,0 +1,219 @@ +======== +Grad-CAM +======== + +This example illustrate how to explain predictions of a Convolutional Neural +Network (CNN) using Grad-CAM. This can be easily achieved with the following +code template snippet: + +.. code-block:: python + + import keras_explainable as ke + + model = tf.keras.applications.ResNet50V2(...) + model = ke.inspection.expose(model) + + scores, cams = ke.gradcam(model, x, y, batch_size=32) + +In this page, we describe how to obtain *Class Activation Maps* (CAMs) from a +trained Convolutional Neural Network (CNN) with respect to an input signal +(an image, in this case) using the Grad-CAM visualization method. +Said maps can be used to explain the model's predictions, determining regions +which most contributed to its effective output. + +Grad-CAM is a form of visualizing regions that most contributed to the output +of a given logit unit of a neural network, often times associated with the +prediction of the occurrence of a class in the problem domain. This method +is first described in the following article: + +Selvaraju, R. R., Cogswell, M., Das, A., Vedantam, R., Parikh, D., & Batra, D. +(2017). Grad-cam: Visual explanations from deep networks via gradient-based +localization. In Proceedings of the IEEE international conference on computer +vision (pp. 618-626). + +Briefly, this can be achieved with the following template snippet: + +.. code-block:: python + + import keras_explainable as ke + + model = build_model(...) + logits, maps = ke.gradients(model, x, y, batch_size=32) + +We describe bellow these lines in detail. + +.. jupyter-execute:: + :hide-code: + :hide-output: + + import os + import numpy as np + import pandas as pd + import tensorflow as tf + from keras.utils import load_img, img_to_array + + import keras_explainable as ke + + SOURCE_DIRECTORY = '_static/images/' + SAMPLES = 8 + SIZES = (299, 299) + + file_names = os.listdir(SOURCE_DIRECTORY) + image_paths = [os.path.join(SOURCE_DIRECTORY, f) + for f in file_names + if f != '_links.txt'] + + images = np.stack([img_to_array(load_img(ip).resize(SIZES)) for ip in image_paths]) + + print('Images shape =', images.shape[1:]) + print('Images avail =', len(images)) + print('Images used =', SAMPLES) + + images = images[:SAMPLES] + +Firstly, we employ the :py:class:`ResNet101` network pre-trained over the +ImageNet dataset: + +.. jupyter-execute:: + + WEIGHTS = 'imagenet' + + input_tensor = tf.keras.Input(shape=(*SIZES, 3), name='inputs') + + rn101 = tf.keras.applications.ResNet101V2( + input_tensor=input_tensor, + classifier_activation=None, + weights=WEIGHTS + ) + rn101.trainable = False + rn101.compile( + optimizer='sgd', + loss='sparse_categorical_crossentropy', + ) + + prec = tf.keras.applications.resnet_v2.preprocess_input + decode_predictions = tf.keras.applications.resnet_v2.decode_predictions + + print(f'ResNet101 with {WEIGHTS} pre-trained weights loaded.') + print(f"Spatial map sizes: {rn101.get_layer('avg_pool').input.shape}") + +We can feed-foward the samples once and get the predicted classes for each sample. +Besides making sure the model is outputing the expected classes, this step is +required in order to determine the most activating units in the *logits* layer, +which improves performance of the explaining methods. + +.. jupyter-execute:: + + inputs = prec(images.copy()) + logits = rn101.predict(inputs, verbose=0) + indices = np.argsort(logits, axis=-1)[:, ::-1] + + probs = tf.nn.softmax(logits).numpy() + predictions = decode_predictions(probs, top=1) + + explaining_units = indices[:, :1] # Firstmost likely classes. + +Grad-CAM works by computing the differential of an activation function, +usually associated with the prediction of a given class, with respect to pixels +contained in the activation map retrieved from an intermediate convolutional +signal (oftentimes advent from the last convolutional layer). + +CAM-based methods implemented here expect the model to output both logits and +activation signal, so their respective representative tensors are exposed and +the jacobian can be computed from the former with respect to the latter. +Hence, we modify the current `rn101` model --- which only output logits at this +time --- to expose both activation maps and logits signals: + +.. jupyter-execute:: + + rn101_exposed = ke.inspection.expose(rn101) + _, cams = ke.gradcam(rn101_exposed, inputs, explaining_units) + + ke.utils.visualize( + images.astype(np.uint8), + overlay=cams.clip(0., 1.).transpose((3, 0, 1, 2)).reshape(-1, *SIZES, 1), + cols=4 + ) + +.. note:: + + To increase efficiency, we sub-select only the top :math:`K` scoring + classification units to explain. The jacobian will only be computed for + these :math:`NK` outputs. + +Breakdown of Model Exposure and Grad-CAM +"""""""""""""""""""""""""""""""""""""""" + +The function :py:func:`keras_explainable.inspection.expose` will take a +:py:class:`keras.Model` as argument and instantiate a new model that outputs +both logits and the activation signal immediately before the +*Global Average Pooling* layer. + +Under the hood of our example, +:py:function:`keras_explainable.inspection.expose` is simply +collecting the input and output signals of the global pooling +and predictions layer, respectively: + +.. code-block:: python + + activations = rn101.get_layer('avg_pool').input + scores = rn101.get_layer('predictions').output + + rn101_exposed = tf.keras.Model(rn101.inputs, [scores, activations]) + +You can also provide hints regarding the argument and output signals, if +your model's topology is more complex or if you simply wish to compute the +Grad-CAM with respect to other layer than the last convolutional one: + +.. code-block:: python + + rn101_exposed = ke.inspection.expose(rn101, 'conv5_out', 'predictions') + +For nested models that were created from different Input objects, you can +further specify which nodes to access within each layer, which maintains +the computation graph connected: + +.. code-block:: python + + from keras import Input, Sequential + from keras.layers import Dense, Activation + from keras.applications import ResNet101V2 + + inputs = Input(shape=[None, None, 3]) + backbone = ResNet101V2(include_top=False, pooling='avg') + model = Sequential([ + inputs, + backbone, + Dense(10, name='logits'), + Activation('softmax', dtype='float32'), + ]) + + rn101_exposed = ke.inspection.expose( + rn101, + arguments={ + 'name': 'rn101.avg_pool', + 'link': 'input', + 'index': 1 + }, + outputs='predictions' + ) + +As for the :py:func:`ke.gradcam` function, it is only a shortcut for +``ke.explain(ke.methods.cams.gradcam, model, inputs, ...)``. + +All explaining methods can also be called directly: + +.. code-block:: python + + gradcam = tf.function(ke.methods.cams.gradcam, reduce_retracing=True) + logits, cams = gradcam(model, inputs, explaining_units) + + cams = ke.filters.positive_normalize(cams) + cams = tf.image.resize(cams, SIZES).numpy() + +Following the original Grad-CAM paper, we only consider the positive +contributing regions in the creation of the CAMs, crunching negatively +contributing and non-related regions together. +This is done automatically by :py:func:`ke.gradcam`, which assigns +the default value :py:func:`filters.positive_normalize` to the +``postprocessing`` parameter. diff --git a/docs/methods/cams/tta_gradcam.rst b/docs/methods/cams/tta_gradcam.rst new file mode 100644 index 0000000..8550bda --- /dev/null +++ b/docs/methods/cams/tta_gradcam.rst @@ -0,0 +1,115 @@ +============ +TTA Grad-CAM +============ + +Test-time augmentation (TTA) is a commonly employed strategy in Saliency +detection and Weakly Supervised Segmentation tasks order to obtain smoother +and more stable explaining maps. + +We illustrate in this example how to apply TTA to AI explaining methods using +``keras-explainable``. This can be easily achieved with the following code +template snippet: + +.. code-block:: python + + import keras_explainable as ke + + model = tf.keras.applications.ResNet50V2(...) + model = ke.inspection.expose(model) + + tta_gradcam = ke.methods.meta.tta( + ke.methods.cams.gradcam, + scales=[0.5, 1.0, 1.5, 2.], + hflip=True + ) + _, cams = ke.explain(tta_gradcam, model, inputs) + +We describe bellow these lines in detail. + +.. jupyter-execute:: + :hide-code: + :hide-output: + + import os + import numpy as np + import pandas as pd + import tensorflow as tf + from keras.utils import load_img, img_to_array + + import keras_explainable as ke + + SOURCE_DIRECTORY = '_static/images/' + SAMPLES = 8 + SIZES = (299, 299) + + file_names = os.listdir(SOURCE_DIRECTORY) + image_paths = [os.path.join(SOURCE_DIRECTORY, f) + for f in file_names + if f != '_links.txt'] + + images = np.stack([img_to_array(load_img(ip).resize(SIZES)) for ip in image_paths]) + + print('Images shape =', images.shape[1:]) + print('Images avail =', len(images)) + print('Images used =', SAMPLES) + + images = images[:SAMPLES] + +Firstly, we employ the :py:class:`ResNet101` network pre-trained over the +ImageNet dataset: + +.. jupyter-execute:: + + WEIGHTS = 'imagenet' + + input_tensor = tf.keras.Input(shape=(*SIZES, 3), name='inputs') + + rn101 = tf.keras.applications.ResNet101V2( + input_tensor=input_tensor, + classifier_activation=None, + weights=WEIGHTS + ) + rn101.trainable = False + rn101.compile( + optimizer='sgd', + loss='sparse_categorical_crossentropy', + ) + + prec = tf.keras.applications.resnet_v2.preprocess_input + decode_predictions = tf.keras.applications.resnet_v2.decode_predictions + + print(f'ResNet101 with {WEIGHTS} pre-trained weights loaded.') + print(f"Spatial map sizes: {rn101.get_layer('avg_pool').input.shape}") + +We can feed-foward the samples once and get the predicted classes for each sample. +Besides making sure the model is outputing the expected classes, this step is +required in order to determine the most activating units in the *logits* layer, +which improves performance of the explaining methods. + +.. jupyter-execute:: + + inputs = prec(images.copy()) + logits = rn101.predict(inputs, verbose=0) + indices = np.argsort(logits, axis=-1)[:, ::-1] + + probs = tf.nn.softmax(logits).numpy() + predictions = decode_predictions(probs, top=1) + + explaining_units = indices[:, :1] # Firstmost likely classes. + +.. jupyter-execute:: + + rn101_exposed = ke.inspection.expose(rn101) + + tta_gradcam = ke.methods.meta.tta( + ke.methods.cams.gradcam, + scales=[0.5, 1.0, 1.5, 2.], + hflip=True + ) + _, cams = ke.explain(tta_gradcam, rn101_exposed, inputs, explaining_units) + + ke.utils.visualize( + images.astype(np.uint8), + overlay=cams.clip(0., 1.).transpose((3, 0, 1, 2)).reshape(-1, *SIZES, 1), + cols=4 + ) diff --git a/docs/methods/index.rst b/docs/methods/index.rst new file mode 100644 index 0000000..e89e67f --- /dev/null +++ b/docs/methods/index.rst @@ -0,0 +1,24 @@ +===================== +AI Explaining Methods +===================== + +In this page, we list the available AI Explaining Methods +and a few examples on how to work with them. + +Saliency and Gradient-based +""""""""""""""""""""""""""" + +.. toctree:: + :maxdepth: 1 + + Gradient Back-propagation + Smooth-Grad + +CAM-Based Techniques +"""""""""""""""""""" + +.. toctree:: + :maxdepth: 1 + + Grad-CAM + TTA Grad-CAM diff --git a/docs/methods/saliency/gradients.rst b/docs/methods/saliency/gradients.rst new file mode 100644 index 0000000..56dbb24 --- /dev/null +++ b/docs/methods/saliency/gradients.rst @@ -0,0 +1,172 @@ +================== +Gradient Back-prop +================== + +In this page, we describe how to obtain *saliency maps* from a trained +Convolutional Neural Network (CNN) with respect to an input signal (an image, +in this case) using the Gradient backprop AI explaining method. +Said maps can be used to explain the model's predictions, determining regions +which most contributed to its effective output. + +Gradient Back-propagation (or Gradient Backprop, for short) is an early +form of visualizing and explaining the salient and contributing features +considered in the decision process of a neural network, being first +described in the following article: + +Simonyan, K., Vedaldi, A., & Zisserman, A. (2013). +Deep inside convolutional networks: Visualising image classification +models and saliency maps. arXiv preprint arXiv:1312.6034. +Available at: https://arxiv.org/abs/1312.6034 + +Briefly, this can be achieved with the following template snippet: + +.. code-block:: python + + import keras_explainable as ke + + model = build_model(...) + model.layers[-1].activation = 'linear' # Usually softmax or sigmoid. + + logits, maps = ke.gradients(model, x, y, batch_size=32) + +We describe bellow these lines in detail. + +.. jupyter-execute:: + :hide-code: + :hide-output: + + import os + import numpy as np + import pandas as pd + import tensorflow as tf + from keras.utils import load_img, img_to_array + + import keras_explainable as ke + + SOURCE_DIRECTORY = '_static/images/' + SAMPLES = 8 + SIZES = (299, 299) + + file_names = os.listdir(SOURCE_DIRECTORY) + image_paths = [os.path.join(SOURCE_DIRECTORY, f) + for f in file_names + if f != '_links.txt'] + + images = np.stack([img_to_array(load_img(ip).resize(SIZES)) for ip in image_paths]) + + print('Images shape =', images.shape[1:]) + print('Images avail =', len(images)) + print('Images used =', SAMPLES) + + images = images[:SAMPLES] + +Firstly, we employ the :py:class:`ResNet101` network pre-trained over the +ImageNet dataset: + +.. jupyter-execute:: + + WEIGHTS = 'imagenet' + + input_tensor = tf.keras.Input(shape=(*SIZES, 3), name='inputs') + + rn101 = tf.keras.applications.ResNet101V2( + input_tensor=input_tensor, + classifier_activation=None, + weights=WEIGHTS + ) + rn101.trainable = False + rn101.compile( + optimizer='sgd', + loss='sparse_categorical_crossentropy', + ) + + prec = tf.keras.applications.resnet_v2.preprocess_input + decode_predictions = tf.keras.applications.resnet_v2.decode_predictions + + print(f'ResNet101 with {WEIGHTS} pre-trained weights loaded.') + print(f"Spatial map sizes: {rn101.get_layer('avg_pool').input.shape}") + +We can feed-foward the samples once and get the predicted classes for each sample. +Besides making sure the model is outputing the expected classes, this step is +required in order to determine the most activating units in the *logits* layer, +which improves performance of the explaining methods. + +.. jupyter-execute:: + + inputs = prec(images.copy()) + logits = rn101.predict(inputs, verbose=0) + + indices = np.argsort(logits, axis=-1)[:, ::-1] + probs = tf.nn.softmax(logits).numpy() + predictions = decode_predictions(probs, top=1) + +.. jupyter-execute:: + :hide-code: + + pd.DataFrame(sum(predictions, []), columns=['code', 'class', 'confidence']) + +Gradient Backprop can be obtained by computing the differential of a function +(usually expressing the logit score for a given class) with respect to pixels +contained in the input signal (usually expressing an image): + +.. jupyter-execute:: + + explaining_units = indices[:, :1] # First most likely class. + + logits, maps = ke.gradients(rn101, inputs, explaining_units) + + ke.utils.visualize(sum(zip(images.astype(np.uint8), maps), ()), cols=4) + +.. note:: + + If the parameter ``indices`` in ``gradients`` is not set, an + explanation for each unit in the explaining layer will be provided, + possibly resuting in *OOM* errors for models containing many units. + + To increase efficiency, we sub-select only the top :math:`K` scoring + classification units to explain. The jacobian will only be computed + for these :math:`NK` outputs. + +Inside the hood, :func:`keras_explainable.gradients` is simply +executing the following call to the +:func:`explain` function: + +.. code-block:: python + + logits, maps = ke.explain( + methods.gradient.gradients, + rn101, + inputs, + explaining_units, + postprocessing=filters.absolute_normalize, + ) + +Following Gradient Backprop paper, we consider the positive and +negative contributing regions in the creation of the saliency maps +by computing their individual absolute contributions before +normalizing them. Different strategies can be employed by +changing the :python:`postprocessing` parameter. + +.. note:: + + For more information on the :func:`explain` function, + check its documentation or its own examples page. + +Of course, we can obtain the same result by directly +calling the :func:`methods.gradient.gradients` function (though it will +not laverage the model's inner distributed strategy and data optimizations +implemented in :func:`explaining.explain`): + +.. jupyter-execute:: + + gradients = tf.function(ke.methods.gradient.gradients, jit_compile=True, reduce_retracing=True) + _, direct_maps = gradients(rn101, inputs, explaining_units) + + direct_maps = ke.filters.absolute_normalize(maps) + direct_maps = tf.image.resize(direct_maps, inputs.shape[1:-1]) + direct_maps = direct_maps.numpy() + + np.testing.assert_array_almost_equal(maps, direct_maps) + print('Maps computed with `explain` and `methods.gradient.gradients` are the same!') + + del logits, direct_maps diff --git a/docs/methods/saliency/smoothgrad.rst b/docs/methods/saliency/smoothgrad.rst new file mode 100644 index 0000000..836522f --- /dev/null +++ b/docs/methods/saliency/smoothgrad.rst @@ -0,0 +1,146 @@ +=========== +Smooth-Grad +=========== + +In this page, we describe how to obtain *saliency maps* from a trained +Convolutional Neural Network (CNN) with respect to an input signal (an image, +in this case) using the Smooth-Grad AI explaining method. + +Smooth-Grad is the variant of the Gradient Backprop algorithm first described +in the following paper: + +Smilkov, D., Thorat, N., Kim, B., Viégas, F., & Wattenberg, M. (2017). +Smoothgrad: removing noise by adding noise. arXiv preprint arXiv:1706.03825. +Available at: https://arxiv.org/abs/1706.03825 + +It consists of consecutive repetitions of the Gradient Backprop method, +each of which is applied over the original sample tempered with +some gaussian noise. +Finally, averaging the resulting explaining maps results in cleaner +visualization results, robust against marginal noise. + +Briefly, this can be achieved with the following template snippet: + +.. code-block:: python + + import keras_explainable as ke + + model = build_model(...) + model.layers[-1].activation = 'linear' # Usually softmax or sigmoid. + + smoothgrad = ke.methods.meta.smooth( + ke.methods.gradient.gradients, + repetitions=10, + noise=0.1 + ) + + logits, maps = ke.explain( + smoothgrad, + model, x, y, + batch_size=32, + postprocessing=ke.filters.absolute_normalize, + ) + +We describe bellow these lines in detail. + +.. jupyter-execute:: + :hide-code: + :hide-output: + + import os + import numpy as np + import pandas as pd + import tensorflow as tf + from keras.utils import load_img, img_to_array + + import keras_explainable as ke + + SOURCE_DIRECTORY = '_static/images/' + SAMPLES = 8 + SIZES = (299, 299) + + file_names = os.listdir(SOURCE_DIRECTORY) + image_paths = [os.path.join(SOURCE_DIRECTORY, f) + for f in file_names + if f != '_links.txt'] + + images = np.stack([img_to_array(load_img(ip).resize(SIZES)) for ip in image_paths]) + + print('Images shape =', images.shape[1:]) + print('Images avail =', len(images)) + print('Images used =', SAMPLES) + + images = images[:SAMPLES] + +Firstly, we employ the :py:class:`ResNet101` network pre-trained over the +ImageNet dataset: + +.. jupyter-execute:: + + WEIGHTS = 'imagenet' + + input_tensor = tf.keras.Input(shape=(*SIZES, 3), name='inputs') + + rn101 = tf.keras.applications.ResNet101V2( + input_tensor=input_tensor, + classifier_activation=None, + weights=WEIGHTS + ) + rn101.trainable = False + rn101.compile( + optimizer='sgd', + loss='sparse_categorical_crossentropy', + ) + + prec = tf.keras.applications.resnet_v2.preprocess_input + decode_predictions = tf.keras.applications.resnet_v2.decode_predictions + + print(f'ResNet101 with {WEIGHTS} pre-trained weights loaded.') + print(f"Spatial map sizes: {rn101.get_layer('avg_pool').input.shape}") + +We can feed-foward the samples once and get the predicted classes for each sample. +Besides making sure the model is outputing the expected classes, this step is +required in order to determine the most activating units in the *logits* layer, +which improves performance of the explaining methods. + +.. jupyter-execute:: + + inputs = prec(images.copy()) + logits = rn101.predict(inputs, verbose=0) + + indices = np.argsort(logits, axis=-1)[:, ::-1] + probs = tf.nn.softmax(logits).numpy() + predictions = decode_predictions(probs, top=1) + + explaining_units = indices[:, :1] # First most likely class. + +keras-explainable implements the Smooth-Grad with the meta explaining function +:func:`keras_explainable.methods.meta.smooth`, which means it wraps any +explaining method and smooths out its outputs. For example: + +.. jupyter-execute:: + + smoothgrad = ke.methods.meta.smooth( + ke.methods.gradient.gradients, + repetitions=40, + noise=0.1, + ) + + _, smoothed_maps = ke.explain( + smoothgrad, + rn101, + inputs, + explaining_units, + postprocessing=ke.filters.absolute_normalize, + ) + +For comparative purposes, we also compute the vanilla gradients method: + +.. jupyter-execute:: + + _, maps = ke.gradients(rn101, inputs, explaining_units) + + ke.utils.visualize( + sum(zip(images.astype(np.uint8), maps, smoothed_maps), ()), + cols=3 + ) diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..d838d38 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +# Requirements file for ReadTheDocs, check .readthedocs.yml. +# To build the module reference correctly, make sure every external package +# under `install_requires` in `setup.cfg` is also listed here! +sphinx>=3.2.1 +sphinx-book-theme +jupyter-sphinx +keras_explainable diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..89a5bed --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +# AVOID CHANGING REQUIRES: IT WILL BE UPDATED BY PYSCAFFOLD! +requires = ["setuptools>=46.1.0", "setuptools_scm[toml]>=5"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +# For smarter version schemes and other configuration options, +# check out https://github.com/pypa/setuptools_scm +version_scheme = "no-guess-dev" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..507d1e8 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,124 @@ +# This file is used to configure your project. +# Read more about the various options under: +# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html +# https://setuptools.pypa.io/en/latest/references/keywords.html + +[metadata] +name = keras-explainable +description = Explainable algorithms for Keras models +author = Lucas David +author_email = lucasolivdavid@gmail.com +license = Apache-2.0 +license_files = LICENSE +long_description = file: README.rst +long_description_content_type = text/x-rst; charset=UTF-8 +url = https://github.com/lucasdavid/keras-explainable + +project_urls = + Documentation = https://pyscaffold.org/ +# Source = https://github.com/pyscaffold/pyscaffold/ +# Changelog = https://pyscaffold.org/en/latest/changelog.html +# Tracker = https://github.com/pyscaffold/pyscaffold/issues +# Conda-Forge = https://anaconda.org/conda-forge/pyscaffold +# Download = https://pypi.org/project/PyScaffold/#files +# Twitter = https://twitter.com/PyScaffold + +# Change if running only on Windows, Mac or Linux (comma-separated) +platforms = any + +# Add here all kinds of additional classifiers as defined under +# https://pypi.org/classifiers/ +classifiers = + Development Status :: 4 - Beta + Programming Language :: Python + +[options] +zip_safe = False +packages = find_namespace: +include_package_data = True +package_dir = + =src + +# Require a min/specific Python version (comma-separated conditions) +# python_requires = >=3.8 + +# Add here dependencies of your project (line-separated), e.g. requests>=2.2,<3.0. +# Version specifiers like >=2.2,<3.0 avoid problems due to API changes in +# new major versions. This works if the required packages follow Semantic Versioning. +# For more information, check out https://semver.org/. +install_requires = + importlib-metadata; python_version<"3.8" + tensorflow + keras + +[options.packages.find] +where = src +exclude = + tests + +[options.extras_require] +# Add here additional requirements for extra features, to install with: +# `pip install keras-explainable[PDF]` like: +# PDF = ReportLab; RXP + +# Add here test requirements (semicolon/line-separated) +testing = + setuptools + pytest + pytest-cov + parameterized + +[options.entry_points] +# Add here console scripts like: +# console_scripts = +# script_name = keras_explainable.module:function +# For example: +# console_scripts = +# fibonacci = keras_explainable.cli:run +# And any other entry points, for example: +# pyscaffold.cli = +# awesome = pyscaffoldext.awesome.extension:AwesomeExtension + +[tool:pytest] +# Specify command line options as you would do when invoking pytest directly. +# e.g. --cov-report html (or xml) for html/xml output or --junitxml junit.xml +# in order to write a coverage file that can be read by Jenkins. +# CAUTION: --cov flags may prohibit setting breakpoints while debugging. +# Comment those flags to avoid this pytest issue. +addopts = + --cov keras_explainable --cov-report term-missing + --verbose +norecursedirs = + dist + build + .tox +testpaths = tests +# Use pytest markers to select/deselect specific tests +# markers = +# slow: mark tests as slow (deselect with '-m "not slow"') +# system: mark end-to-end system tests + +[devpi:upload] +# Options for the devpi: PyPI server and packaging tool +# VCS export must be deactivated since we are using setuptools-scm +no_vcs = 1 +formats = bdist_wheel + +[flake8] +# Some sane defaults for the code style checker flake8 +max_line_length = 90 +extend_ignore = E203, W503 +# ^ Black-compatible +# E203 and W503 have edge cases handled by black +exclude = + .tox + build + dist + .eggs + docs/conf.py + +[pyscaffold] +# PyScaffold's parameters when the project was created. +# This will be used when updating. Do not change! +version = 4.3.1 +package = keras_explainable diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ee09efe --- /dev/null +++ b/setup.py @@ -0,0 +1,21 @@ +""" + Setup file for keras-explainable. + Use setup.cfg to configure your project. + + This file was generated with PyScaffold 4.3.1. + PyScaffold helps you to put up the scaffold of your new Python project. + Learn more under: https://pyscaffold.org/ +""" +from setuptools import setup + +if __name__ == "__main__": + try: + setup(use_scm_version={"version_scheme": "no-guess-dev"}) + except: # noqa + print( + "\n\nAn error occurred while building the project, " + "please ensure you have the most updated version of setuptools, " + "setuptools_scm and wheel with:\n" + " pip install -U setuptools setuptools_scm wheel\n\n" + ) + raise diff --git a/src/keras_explainable/__init__.py b/src/keras_explainable/__init__.py new file mode 100644 index 0000000..d5cd4cb --- /dev/null +++ b/src/keras_explainable/__init__.py @@ -0,0 +1,49 @@ +import sys + +from keras_explainable import methods +from keras_explainable import inspection +from keras_explainable import filters +from keras_explainable import utils +from keras_explainable.engine import explaining +from keras_explainable.engine.explaining import explain, partial_explain + +if sys.version_info[:2] >= (3, 8): + # TODO: Import directly (no need for conditional) when `python_requires = >= 3.8` + from importlib.metadata import PackageNotFoundError # pragma: no cover + from importlib.metadata import version +else: + from importlib_metadata import PackageNotFoundError # pragma: no cover + from importlib_metadata import version + +try: + # Change here if project is renamed and does not equal the package name + dist_name = "keras-explainable" + __version__ = version(dist_name) +except PackageNotFoundError: # pragma: no cover + __version__ = "unknown" +finally: + del version, PackageNotFoundError + +cam = partial_explain(methods.cams.cam, postprocessing=filters.positive_normalize) +gradcam = partial_explain(methods.cams.gradcam, postprocessing=filters.positive_normalize) +gradcampp = partial_explain(methods.cams.gradcampp, postprocessing=filters.positive_normalize) +scorecam = partial_explain(methods.cams.scorecam, postprocessing=filters.positive_normalize) + +gradients = partial_explain( + methods.gradient.gradients, + postprocessing=filters.absolute_normalize, +) + +__all__ = [ + "methods", + "inspection", + "filters", + "utils", + "explaining", + "explain", + "cam", + "gradcam", + "gradcampp", + "scorecam", + "gradients", +] diff --git a/src/keras_explainable/engine/__init__.py b/src/keras_explainable/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/keras_explainable/engine/explaining.py b/src/keras_explainable/engine/explaining.py new file mode 100644 index 0000000..9e25316 --- /dev/null +++ b/src/keras_explainable/engine/explaining.py @@ -0,0 +1,256 @@ +import warnings +from typing import Any, Callable, Dict, Optional, Tuple, Union + +import numpy as np +import tensorflow as tf +from keras import callbacks as callbacks_module +from keras.engine import data_adapter +from keras.engine.training import ( + _is_tpu_multi_host, + _minimum_control_deps, + potentially_ragged_concat, + reduce_per_replica +) +from keras.utils import tf_utils +from tensorflow.python.eager import context + +from keras_explainable.inspection import SPATIAL_AXIS + +def explain_step( + model: tf.keras.Model, + method: Callable, + data: Tuple[tf.Tensor], + spatial_axis: Tuple[int, int] = SPATIAL_AXIS, + postprocessing: Callable = None, + resizing: Optional[Union[bool, tf.Tensor]] = True, + **params, +) -> Tuple[tf.Tensor, tf.Tensor]: + inputs, indices, _ = data_adapter.unpack_x_y_sample_weight(data) + logits, maps = method( + model=model, + inputs=inputs, + indices=indices, + spatial_axis=spatial_axis, + **params, + ) + + if postprocessing is not None: + maps = postprocessing(maps, axis=spatial_axis) + + if resizing is not None and resizing is not False: + if resizing is True: + resizing = tf.shape(inputs)[1:-1] + maps = tf.image.resize(maps, resizing) + + return logits, maps + +def make_explain_function( + model: tf.keras.Model, + method: Callable, + params: Dict[str, Any], + force: bool = False, +): + explain_function = getattr(model, 'explain_function', None) + + if explain_function is not None and not force: + return explain_function + + def explain_function(iterator): + """Runs a single explain step.""" + + def run_step(data): + outputs = explain_step(model, method, data, **params) + # Ensure counter is updated only if `test_step` succeeds. + with tf.control_dependencies(_minimum_control_deps(outputs)): + model._explain_counter.assign_add(1) + return outputs + + if model._jit_compile: + run_step = tf.function(run_step, jit_compile=True, reduce_retracing=True) + + data = next(iterator) + outputs = model.distribute_strategy.run(run_step, args=(data,)) + outputs = reduce_per_replica( + outputs, model.distribute_strategy, reduction="concat" + ) + return outputs + + if not model.run_eagerly: + explain_function = tf.function(explain_function, reduce_retracing=True) + + model.explain_function = explain_function + + return explain_function + +def make_data_handler( + model, + x, + y, + batch_size=None, + steps=None, + max_queue_size=10, + workers=1, + use_multiprocessing=False, +): + dataset_types = (tf.compat.v1.data.Dataset, tf.data.Dataset) + if ( + model._in_multi_worker_mode() + or _is_tpu_multi_host(model.distribute_strategy) + ) and isinstance(x, dataset_types): + try: + opts = tf.data.Options() + opts.experimental_distribute.auto_shard_policy = ( + tf.data.experimental.AutoShardPolicy.DATA + ) + x = x.with_options(opts) + except ValueError: + warnings.warn( + "Using evaluate with MultiWorkerMirroredStrategy " + "or TPUStrategy and AutoShardPolicy.FILE might lead to " + "out-of-order result. Consider setting it to " + "AutoShardPolicy.DATA.", + stacklevel=2, + ) + + return data_adapter.get_data_handler( + x=x, + y=y, + batch_size=batch_size, + steps_per_epoch=steps, + initial_epoch=0, + epochs=1, + max_queue_size=max_queue_size, + workers=workers, + use_multiprocessing=use_multiprocessing, + model=model, + steps_per_execution=model._steps_per_execution, + ) + +def explain( + method: Callable, + model: tf.keras.Model, + x: Union[np.ndarray, tf.Tensor, tf.data.Dataset], + y: Optional[Union[np.ndarray, tf.Tensor]] = None, + batch_size=None, + verbose="auto", + steps=None, + callbacks=None, + max_queue_size=10, + workers=1, + use_multiprocessing=False, + force: bool = True, + **method_params, +): + """Explain the output with respect to an intermediate signal. + + :param method: An AI explaining function, as the ones contained in `methods` module. + :param model: The `tf.keras.Model` whose predictions should be explained. + :param x: the input data for the model. + :param y: the indices in the output tensor that should be explained. + If none, an activation map is computed for each unit. + :param indices_batch_dims: The dimensions set as `batch` when gathering units + described by `indices`. Ignored if `indices` is None. + :param indices_axis: The axis from which to gather units described by `indices`. + Ignored if `indices` is None. + :param spatial_axis: The axes containing the positional visual info. We + assume `inputs` to contain 2D images or videos in the shape + `(B1, B2, ..., BN, H, W, 3)`. For 3D image data, set + `spatial_axis` to `(1, 2, 3)` or `(-4, -3, -2)`. + :param postprocessing: A function to process the activation maps before + normalization (most commonly adopted being `maximum(x, 0)` and + `abs`). + + :return: + Logits: the output signal of the model, collected from `logits_layer`. + Maps: the activation maps produced by Grad-CAM that explain the logits + with respect to the intermediate positional signal in the model. + """ + + if not hasattr(model, '_explain_counter'): + agg = tf.VariableAggregation.ONLY_FIRST_REPLICA + model._explain_counter = tf.Variable(0, dtype="int64", aggregation=agg) + + outputs = None + with model.distribute_strategy.scope(): + # Creates a `tf.data.Dataset` and handles batch and epoch iteration. + data_handler = make_data_handler( + model, + x, + y, + batch_size=batch_size, + steps=steps, + max_queue_size=max_queue_size, + workers=workers, + use_multiprocessing=use_multiprocessing, + ) + + # Container that configures and calls `tf.keras.Callback`s. + if not isinstance(callbacks, callbacks_module.CallbackList): + callbacks = callbacks_module.CallbackList( + callbacks, + add_history=True, + add_progbar=verbose != 0, + model=model, + verbose=verbose, + epochs=1, + steps=data_handler.inferred_steps, + ) + + explain_function = make_explain_function( + model, method, method_params, force + ) + model._explain_counter.assign(0) + callbacks.on_predict_begin() + batch_outputs = None + for _, iterator in data_handler.enumerate_epochs(): # Single epoch. + with data_handler.catch_stop_iteration(): + for step in data_handler.steps(): + callbacks.on_predict_batch_begin(step) + tmp_batch_outputs = explain_function(iterator) + if data_handler.should_sync: + context.async_wait() + batch_outputs = ( + tmp_batch_outputs # No error, now safe to assign. + ) + if outputs is None: + outputs = tf.nest.map_structure( + lambda batch_output: [batch_output], + batch_outputs, + ) + else: + tf.__internal__.nest.map_structure_up_to( + batch_outputs, + lambda output, batch_output: output.append( + batch_output + ), + outputs, + batch_outputs, + ) + end_step = step + data_handler.step_increment + callbacks.on_predict_batch_end( + end_step, {"outputs": batch_outputs} + ) + if batch_outputs is None: + raise ValueError( + "Unexpected result of `explain_function` " + "(Empty batch_outputs). Please use " + "`Model.compile(..., run_eagerly=True)`, or " + "`tf.config.run_functions_eagerly(True)` for more " + "information of where went wrong, or file a " + "issue/bug to `keras-explainable`." + ) + callbacks.on_predict_end() + all_outputs = tf.__internal__.nest.map_structure_up_to( + batch_outputs, potentially_ragged_concat, outputs + ) + return tf_utils.sync_to_numpy_or_python_type(all_outputs) + +def partial_explain(method, **default_params): + + def _partial_method_explain(*args, **params): + params = {**default_params, **params} + return explain(method, *args, **params) + + _partial_method_explain.__name__ = f'{method.__name__}_explain' + + return _partial_method_explain diff --git a/src/keras_explainable/filters.py b/src/keras_explainable/filters.py new file mode 100644 index 0000000..c9ecf7c --- /dev/null +++ b/src/keras_explainable/filters.py @@ -0,0 +1,25 @@ +import tensorflow as tf + +from keras_explainable.inspection import SPATIAL_AXIS + +def normalize(x, axis=SPATIAL_AXIS): + """Normalize a positional signal between 0 and 1.""" + x = tf.convert_to_tensor(x) + x -= tf.reduce_min(x, axis=axis, keepdims=True) + + return tf.math.divide_no_nan(x, tf.reduce_max(x, axis=axis, keepdims=True)) + +def positive(x, axis=SPATIAL_AXIS): + return tf.nn.relu(x) + +def negative(x, axis=SPATIAL_AXIS): + return tf.nn.relu(-x) + +def positive_normalize(x, axis=SPATIAL_AXIS): + return normalize(positive(x, axis=axis), axis=axis) + +def absolute_normalize(x, axis=SPATIAL_AXIS): + return normalize(tf.abs(x), axis=axis) + +def negative_normalize(x, axis=SPATIAL_AXIS): + return normalize(negative(x), axis=axis) diff --git a/src/keras_explainable/inspection.py b/src/keras_explainable/inspection.py new file mode 100644 index 0000000..d2587dd --- /dev/null +++ b/src/keras_explainable/inspection.py @@ -0,0 +1,319 @@ +"""Inspection utils for models and layers. +""" + +from typing import Dict, List, Optional, Tuple, Type, Union + +import tensorflow as tf +from keras.engine.base_layer import Layer +from keras.engine.keras_tensor import KerasTensor +from keras.engine.training import Model +from keras.layers.normalization.batch_normalization import BatchNormalizationBase +from keras.layers.normalization.layer_normalization import LayerNormalization +from keras.layers.pooling.base_global_pooling1d import GlobalPooling1D +from keras.layers.pooling.base_global_pooling2d import GlobalPooling2D +from keras.layers.pooling.base_global_pooling3d import GlobalPooling3D + +from keras_explainable.utils import tolist + +E = Union[str, int, tf.Tensor, KerasTensor, Dict[str, Union[str, int]]] + +KERNEL_AXIS = -1 +SPATIAL_AXIS = (-3, -2) + +NORMALIZATION_LAYERS = ( + BatchNormalizationBase, + LayerNormalization, +) + +POOLING_LAYERS = ( + GlobalPooling1D, + GlobalPooling2D, + GlobalPooling3D, +) + +def get_nested_layer( + model: Model, + name: str, +) -> Layer: + """Retrieve a nested layer in the model. + + Args: + model (Model): the model containing the nested layer. + name (str): the descriptor of the nested layer. + Nested layers are separated by "." + + Example: + ```py + model = tf.keras.Sequential([ + tf.keras.applications.ResNet101V2(include_top=False, pooling='avg'), + tf.keras.layers.Dense(10, activation='softmax', name='predictions') + ]) + + pooling_layer = get_nested_layer(model, 'resnet101v2.avg_pool') + ``` + + Raises: + ValueError: if `name` is not a nested member of `model`. + + Returns: + tf.keras.layer.Layer: the retrieved layer. + """ + for n in name.split("."): + model = model.get_layer(n) + + return model + +def get_logits_layer( + model: Model, + name: str = None, +) -> Layer: + """Retrieve the "logits" layer. + + Args: + model (Model): the model containing the logits layer. + name (str, optional): the name of the layer, if known. Defaults to None. + + Raises: + ValueError: if a logits layer cannot be found + + Returns: + Layer: the retrieved logits layer + """ + return find_layer_with(model, name, properties=['kernel']) + +def get_global_pooling_layer( + model: Model, + name: str = None, +) -> Layer: + """Retrieve the last global pooling layer. + + Args: + model (Model): the model containing the pooling layer. + name (str, optional): the name of the layer, if known. Defaults to None. + + Raises: + ValueError: if a pooling layer cannot be found + + Returns: + Layer: the retrieved pooling layer + """ + return find_layer_with(model, name, klass=POOLING_LAYERS) + +def find_layer_with( + model: Model, + name: Optional[str] = None, + properties: Optional[Tuple[str]] = None, + klass: Optional[Tuple[Type[Layer]]] = None, + search_reversed: bool = True, +) -> Layer: + """Find a layer within a model that satisfies all required properties. + + Args: + model (Model): the container model. + name (Optional[str], optional): the name of the layer, if known. + Defaults to None. + properties (Optional[Tuple[str]], optional): a list of properties that + should be visible from the searched layer. Defaults to None. + klass (Optional[Tuple[Type[Layer]]], optional): a collection of classes + allowed for the searched layer. Defaults to None. + search_reversed (bool, optional): wether to search from last-to-first. + Defaults to True. + + Raises: + ValueError: if no search parameters are passed. + ValueError: if no valid layer can be found with the specified search + parameters. + + Returns: + Layer: the layer satisfying all search parameters. + """ + if name == properties == klass == None: + raise ValueError( + 'At least one of the search search parameters must ' + 'be set when calling `get_layer`, indicating the ' + 'necessary properties for the layer being retrieved.' + ) + + if name is not None: + return get_nested_layer(model, name) + + layers = model.layers + if search_reversed: + layers = reversed(layers) + + for layer in layers: + if klass and not isinstance(layer, klass): + continue + if properties and not all(hasattr(layer, p) for p in properties): + continue + + return layer # `layer` matches all conditions. + + raise ValueError( + "A valid layer couldn't be inferred from the name=`{name}`, " + "klass=`{klass}` and properties=`{properties}`. Make sure these " + "attributes correctly reflect a layer in the model." + ) + +def endpoints(model: Model, endpoints: List[E]) -> List[KerasTensor]: + """Collect intermediate endpoints in a model based on structured descriptors. + + Args: + model (Model): the model containing the endpoints to be collected. + endpoints (List[E]): descriptors of endpoints that should be collected. + + Raises: + ValueError: raised whenever one of the endpoint descriptors is invalid + or it does not describe a nested layer in the `model`. + + Returns: + List[KerasTensor]: a list containing the endpoints of interest. + """ + endpoints_ = [] + + for ep in endpoints: + if isinstance(ep, int): + endpoint = model.layers[ep].get_output_at(0) + elif isinstance(ep, Layer): + endpoint = ep.get_output_at(0) + else: + if isinstance(ep, str): + ep = {"name": ep} + + if not isinstance(ep, dict): + raise ValueError( + f"Illegal type {type(ep)} for endpoint {ep}. Expected a " + "layer index (int), layer name (str) or a dictionary with " + "`name`, `link` and `index` keys." + ) + + layer = ep["name"] + link = ep.get("link", "output") + index = ep.get("index", 0) + + endpoint = ( + get_nested_layer(model, layer).get_input_at(index) + if link == "input" + else get_nested_layer(model, layer).get_output_at(index) + ) + + endpoints_.append(endpoint) + + return endpoints_ + +def expose( + model: Model, + arguments: Optional[E] = None, + outputs: Optional[E] = None, +) -> Model: + """Creates a new model that exposes all endpoints described by + `arguments` and `outputs`. + + Args: + model (Model): The model being explained. + arguments (Optional[E], optional): Name of the argument layer/tensor in + the model. The jacobian of the output explaining units will be computed + with respect to the input signal of this layer. This argument can also + be an integer, a dictionary representing the intermediate signal or + the pooling layer itself. If None is passed, the penultimate layer + is assumed to be a GAP layer. Defaults to None. + outputs (Optional[E], optional): Name of the output layer in the model. + The jacobian will be computed for the activation signal of units in this + layer. This argument can also be an integer, a dictionary representing + the output signal and the logits layer itself. If None is passed, + the last layer is assumed to be the logits layer.. Defaults to None. + + Returns: + Model: the exposed model, whose outputs contain the intermediate + and output tensors. + """ + if outputs is None: + outputs = get_logits_layer(model) + if arguments is None: + arguments = get_global_pooling_layer(model).name + if isinstance(arguments, str): + arguments = {"name": arguments, "link": "input"} + + outputs = tolist(outputs) + arguments = tolist(arguments) + + tensors = endpoints(model, outputs + arguments) + + return Model( + inputs=model.inputs, + outputs=tensors, + ) + +def gather_units( + tensor: tf.Tensor, + indices: Optional[tf.Tensor], + axis: int = -1, + batch_dims: int = -1, +) -> tf.Tensor: + """Gather units (in the last axis) from a tensor. + + Args: + tensor (tf.Tensor): the input tensor. + indices (tf.Tensor, optional): the indices that should be gathered. + axis (int, optional): the axis from which indices should be taken, + used to fine control gathering. Defaults to -1. + batch_dims (int, optional): the number of batch dimensions, used to + fine control gathering. Defaults to -1. + + Returns: + tf.Tensor: the gathered units + """ + if indices is None: + return tensor + + return tf.gather(tensor, indices, axis=axis, batch_dims=batch_dims) + +def layers_with_biases( + model: Model, + exclude: Tuple[Layer] = (), + return_biases: bool = True, +) -> List[Layer]: + layers = [ + layer + for layer in model._flatten_layers(include_self=False) + if ( + layer not in exclude + and ( + isinstance(layer, NORMALIZATION_LAYERS) + or hasattr(layer, 'bias') and layer.bias is not None + ) + ) + ] + + if return_biases: + return layers, biases(layers) + + return layers + +def biases( + layers: List[Layer], +) -> List[tf.Tensor]: + """Retrieve all biases from a model. + + Args: + model (Model): the model being inspected. + + Returns: + List[tf.Tensor]: a list of all biases retrieved. + """ + biases = [] + + for layer in layers: + if isinstance(layer, NORMALIZATION_LAYERS): + # Batch norm := ((x - m)/s)*w + b + # Hence bias factor is -m*w/s + b. + biases.append( + -layer.moving_mean * layer.gamma + / tf.sqrt(layer.moving_variance + 1e-07) # might be variance here. + + layer.beta + ) + + elif hasattr(layer, 'bias') and layer.bias is not None: + biases.append(layer.bias) + + return biases diff --git a/src/keras_explainable/methods/__init__.py b/src/keras_explainable/methods/__init__.py new file mode 100644 index 0000000..36524a6 --- /dev/null +++ b/src/keras_explainable/methods/__init__.py @@ -0,0 +1,9 @@ +from keras_explainable.methods import cams +from keras_explainable.methods import gradient +from keras_explainable.methods import meta + +__all__ = [ + "cams", + "gradient", + "meta", +] diff --git a/src/keras_explainable/methods/cams.py b/src/keras_explainable/methods/cams.py new file mode 100644 index 0000000..7711f73 --- /dev/null +++ b/src/keras_explainable/methods/cams.py @@ -0,0 +1,196 @@ +from typing import Optional, Tuple + +import tensorflow as tf +from keras.backend import int_shape + +from keras_explainable.filters import normalize +from keras_explainable.inspection import KERNEL_AXIS +from keras_explainable.inspection import SPATIAL_AXIS +from keras_explainable.inspection import get_logits_layer +from keras_explainable.inspection import gather_units + +METHODS = [] + +def cam( + model: tf.keras.Model, + inputs: tf.Tensor, + indices: Optional[tf.Tensor] = None, + indices_axis: int = KERNEL_AXIS, + indices_batch_dims: int = -1, + spatial_axis: Tuple[int] = SPATIAL_AXIS, + logits_layer: Optional[str] = None, +): + """Computes the CAM Visualization Method. + + This method expects `inputs` to be a batch of positional signals of shape + `BHWC`, and will return a tensor of shape `BH'W'L`, where `(H', W')` are + the sizes of the visual receptive field in the explained activation layer + and `L` is the number of labels represented within the model's output + logits. + + If `indices` is passed, the specific logits indexed by elements in this + tensor are selected before the gradients are computed, effectivelly + reducing the columns in the jacobian, and the size of the output + explaining map. + + """ + + logits, activations = model(inputs, training=False) + logits = gather_units(logits, indices, indices_axis, indices_batch_dims) + + weights = get_logits_layer(model, name=logits_layer).kernel + weights = gather_units(weights, indices, axis=-1, batch_dims=0) + + dims = ("kc" if indices is None else "kbc") + maps = tf.einsum(f"b...k,{dims}->b...c", activations, weights) + + return logits, maps + +def gradcam( + model: tf.keras.Model, + inputs: tf.Tensor, + indices: Optional[tf.Tensor] = None, + indices_axis: int = KERNEL_AXIS, + indices_batch_dims: int = -1, + spatial_axis: Tuple[int] = SPATIAL_AXIS, +): + """Computes the Grad-CAM Visualization Method. + + This method expects `inputs` to be a batch of positional signals of shape + `BHWC`, and will return a tensor of shape `BH'W'L`, where `(H', W')` are + the sizes of the visual receptive field in the explained activation layer + and `L` is the number of labels represented within the model's output + logits. + + If `indices` is passed, the specific logits indexed by elements in this + tensor are selected before the gradients are computed, effectivelly + reducing the columns in the jacobian, and the size of the output + explaining map. + + References: + + - Selvaraju, R. R., Cogswell, M., Das, A., Vedantam, R., Parikh, D., & Batra, D. + (2017). Grad-cam: Visual explanations from deep networks via gradient-based + localization. In Proceedings of the IEEE international conference on computer + vision (pp. 618-626). + + """ + with tf.GradientTape(watch_accessed_variables=False) as tape: + tape.watch(inputs) + logits, activations = model(inputs, training=False) + logits = gather_units(logits, indices, indices_axis, indices_batch_dims) + + dlda = tape.batch_jacobian(logits, activations) + weights = tf.reduce_mean(dlda, axis=spatial_axis) + maps = tf.einsum("b...k,bck->b...c", activations, weights) + + return logits, maps + +def gradcampp( + model: tf.keras.Model, + inputs: tf.Tensor, + indices: Optional[tf.Tensor] = None, + indices_axis: int = KERNEL_AXIS, + indices_batch_dims: int = -1, + spatial_axis: Tuple[int] = SPATIAL_AXIS, +): + """Computes the Grad-CAM++ Visualization Method. + + This method expects `inputs` to be a batch of positional signals of shape + `BHWC`, and will return a tensor of shape `BH'W'L`, where `(H', W')` are + the sizes of the visual receptive field in the explained activation layer + and `L` is the number of labels represented within the model's output + logits. + + If `indices` is passed, the specific logits indexed by elements in this + tensor are selected before the gradients are computed, effectivelly + reducing the columns in the jacobian, and the size of the output + explaining map. + + References: + + - Chattopadhay, A., Sarkar, A., Howlader, P., & Balasubramanian, V. N. (2018, March). + Grad-cam++: Generalized gradient-based visual explanations for deep convolutional + networks. In 2018 IEEE winter conference on applications of computer vision + (WACV) (pp. 839-847). IEEE. + + - Grad-CAM++'s official implementation. Github. + Available at: https://github.com/adityac94/Grad_CAM_plus_plus. + + """ + with tf.GradientTape(watch_accessed_variables=False) as tape: + tape.watch(inputs) + logits, activations = model(inputs, training=False) + logits = gather_units(logits, indices, indices_axis, indices_batch_dims) + + dlda = tape.batch_jacobian(logits, activations) + + dyda = tf.einsum('bc,bc...k->bc...k', tf.exp(logits), dlda) + d2 = dlda**2 + d3 = dlda**3 + aab = tf.reduce_sum(activations, axis=spatial_axis) # (BK) + akc = tf.math.divide_no_nan( + d2, + 2.*d2 + tf.einsum('bk,bc...k->bc...k', aab, d3) # (2*(BUHWK) + (BK)*BUHWK) + ) + + # Tensorflow has a glitch that doesn't allow this form: + # weights = tf.einsum('bc...k,bc...k->bck', akc, tf.nn.relu(dyda)) # w: buk + # So we use this one instead: + weights = tf.reduce_sum(akc * tf.nn.relu(dyda), axis=spatial_axis) + + maps = tf.einsum('bck,b...k->b...c', weights, activations) # a: bhwk, m: buhw + + return logits, maps + +def scorecam( + model: tf.keras.Model, + inputs: tf.Tensor, + indices: Optional[tf.Tensor] = None, + indices_axis: int = KERNEL_AXIS, + indices_batch_dims: int = -1, + spatial_axis: Tuple[int] = SPATIAL_AXIS, +): + """Computes the Score-CAM Visualization Method. + + This method expects `inputs` to be a batch of positional signals of shape + `BHWC`, and will return a tensor of shape `BH'W'L`, where `(H', W')` are + the sizes of the visual receptive field in the explained activation layer + and `L` is the number of labels represented within the model's output + logits. + + If `indices` is passed, the specific logits indexed by elements in this + tensor are selected before the gradients are computed, effectivelly + reducing the columns in the jacobian, and the size of the output + explaining map. + + References: + + - Score-CAM: Score-Weighted Visual Explanations for Convolutional Neural + Networks. Available at: https://arxiv.org/abs/1910.01279 + + """ + + scores, activations = model(inputs, training=False) + scores = gather_units(scores, indices, indices_axis, indices_batch_dims) + + classes = int_shape(scores)[-1] or tf.shape(scores)[-1] + kernels = int_shape(activations)[-1] or tf.shape(activations)[-1] + + shape = tf.shape(inputs) + sizes = [shape[a] for a in spatial_axis] + maps = tf.zeros([shape[0]] + sizes + [classes]) + + for i in tf.range(kernels): + mask = activations[..., i:i+1] + mask = normalize(mask, axis=spatial_axis) + mask = tf.image.resize(mask, sizes) + + si, _ = model(inputs * mask, training=False) + si = gather_units(si, indices, indices_axis, indices_batch_dims) + si = tf.einsum('bc,bhw->bhwc', si, mask[..., 0]) + maps += si + + return scores, maps + +METHODS.extend((cam, gradcam, gradcampp, scorecam)) diff --git a/src/keras_explainable/methods/gradient.py b/src/keras_explainable/methods/gradient.py new file mode 100644 index 0000000..706c468 --- /dev/null +++ b/src/keras_explainable/methods/gradient.py @@ -0,0 +1,148 @@ +from functools import partial +from typing import List, Optional, Tuple + +import tensorflow as tf +from keras_explainable import filters +from keras_explainable.inspection import KERNEL_AXIS +from keras_explainable.inspection import SPATIAL_AXIS +from keras_explainable.inspection import gather_units +from keras_explainable.inspection import biases + +METHODS = [] + +def transpose_jacobian(x, spatial_rank=len(SPATIAL_AXIS)): + dims = [2 + i for i in range(spatial_rank)] + + return tf.transpose(x, [0] + dims + [1]) + +def gradients( + model: tf.keras.Model, + inputs: tf.Tensor, + indices: Optional[tf.Tensor] = None, + indices_axis: int = KERNEL_AXIS, + indices_batch_dims: int = -1, + spatial_axis: Tuple[int] = SPATIAL_AXIS, +): + """Computes the Grad-CAM Visualization Method. + + This method expects `inputs` to be a batch of positional signals of shape + `BHWC`, and will return a tensor of shape `BH'W'L`, where `(H', W')` are + the sizes of the visual receptive field in the explained activation layer + and `L` is the number of labels represented within the model's output + logits. + + If `indices` is passed, the specific logits indexed by elements in this + tensor are selected before the gradients are computed, effectivelly + reducing the columns in the jacobian, and the size of the output + explaining map. + + References: + + - Simonyan, K., Vedaldi, A., & Zisserman, A. (2013). + Deep inside convolutional networks: Visualising image classification + models and saliency maps. arXiv preprint arXiv:1312.6034. + + """ + + with tf.GradientTape(watch_accessed_variables=False) as tape: + tape.watch(inputs) + logits = model(inputs, training=False) + logits = gather_units(logits, indices, indices_axis, indices_batch_dims) + + maps = tape.batch_jacobian(logits, inputs) + maps = tf.reduce_mean(maps, axis=-1) + maps = transpose_jacobian(maps, len(spatial_axis)) + + return logits, maps + +METHODS.extend((gradients,)) + +def resized_psi_dfx( + inputs: tf.Tensor, + outputs: tf.Tensor, + sizes: tf.Tensor, + psi: callable = filters.absolute_normalize, + spatial_axis: Tuple[int] = SPATIAL_AXIS, +) -> tf.Tensor: + t = outputs * inputs + t = psi(t, spatial_axis) + t = tf.reduce_mean(t, axis=-1, keepdims=True) + # t = transpose_jacobian(t, len(spatial_axis)) + t = tf.image.resize(t, sizes) + + return t + +def full_gradients( + model: tf.keras.Model, + inputs: tf.Tensor, + indices: Optional[tf.Tensor] = None, + indices_axis: int = KERNEL_AXIS, + indices_batch_dims: int = -1, + spatial_axis: Tuple[int] = SPATIAL_AXIS, + psi: callable = filters.absolute_normalize, + biases: Optional[List[tf.Tensor]] = None, + node_index: int = 0, +): + """Computes the Full-Gradient Visualization Method. + + As described in the article "Full-Gradient Representation forNeural Network + Visualization", Full-Gradient can be summarized in the following equation: + + ::math:: + + f(x) = ψ(∇_xf(x)\odot x) +∑_{l\in L}∑_{c\in c_l} ψ(f^b(x)_c) + + This approach main idea is to add to add the individual contributions of + each bias factor in the network onto the extracted gradient. + + This method expects `inputs` to be a batch of positional signals of shape + `BHWC`, and will return a tensor of shape `BH'W'L`, where `(H', W')` are + the sizes of the visual receptive field in the explained activation layer + and `L` is the number of labels represented within the model's output + logits. + + If `indices` is passed, the specific logits indexed by elements in this + tensor are selected before the gradients are computed, effectivelly + reducing the columns in the jacobian, and the size of the output + explaining map. + + References: + + - Srinivas S, Fleuret F. Full-gradient representation for neural network visualization. + [arXiv preprint arXiv:1905.00780](https://arxiv.org/pdf/1905.00780.pdf), 2019. + + """ + + shape = tf.shape(inputs) + sizes = [shape[a] for a in spatial_axis] + + resized_psi_dfx_ = partial( + resized_psi_dfx, + sizes=sizes, + psi=psi, + spatial_axis=spatial_axis, + ) + + if biases is None: + _, biases = biases( + model, + node_index=node_index, + exclude=tf.keras.layers.Dense + ) + + with tf.GradientTape(watch_accessed_variables=False) as tape: + tape.watch(inputs) + logits, *intermediates = model(inputs, training=False) + logits = gather_units(logits, indices, indices_axis, indices_batch_dims) + + maps, *intermediate_maps = tape.gradient(logits, [inputs, *intermediates]) + + maps = resized_psi_dfx_(inputs, maps) + for b, i in zip(biases, intermediate_maps): + maps += resized_psi_dfx_(b, i) + # for idx in tf.range(len(biases)): + # maps += resized_psi_dfx_(biases[idx], intermediate_maps[idx]) + + return logits, maps + +METHODS.extend((full_gradients,)) diff --git a/src/keras_explainable/methods/meta.py b/src/keras_explainable/methods/meta.py new file mode 100644 index 0000000..d2f6512 --- /dev/null +++ b/src/keras_explainable/methods/meta.py @@ -0,0 +1,129 @@ +from functools import partial +from typing import Callable, List, Tuple + +import tensorflow as tf + +from keras_explainable.inspection import SPATIAL_AXIS + +def smooth( + method: Callable, + repetitions: int = 20, + noise: int = 0.1, +): + """Smooth Meta Explaining Method. + + Args: + method (Callable): the explaining method to be smoothed + repetitions (int, optional): number of repetitions. Defaults to 20. + noise (int, optional): standard deviation of the gaussian noise + added to the input signal. Defaults to 0.1. + + References: + + - Smilkov, D., Thorat, N., Kim, B., Viégas, F., & Wattenberg, M. (2017). + Smoothgrad: removing noise by adding noise. arXiv preprint arXiv:1706.03825. + Available at: https://arxiv.org/abs/1706.03825 + """ + def _smooth( + model: tf.keras.Model, + inputs: tf.Tensor, + *args, + **params, + ): + """Computes the Smooth-Grad Visualization Method. + + """ + + outputs = method(model, inputs, *args, **params) + shape = tf.shape(inputs) + + for step in tf.range(repetitions - 1): + noisy_inputs = inputs + tf.random.normal(shape, 0, noise, inputs.dtype) + noisy_outputs = method(model, noisy_inputs, *args, **params) + + for o, n in zip(outputs, noisy_outputs): + o += n + + for o in outputs: + o /= repetitions + + return outputs + + _smooth.__name__ = f'{method.__name__}_smooth' + return _smooth + +def tta( + method: Callable, + scales: List[float] = [0.5, 1.5, 2.], + hflip: bool = True, + resize_method: str = 'bilinear', +): + """Computes the TTA version of a visualization method. + + """ + scales = tf.convert_to_tensor(scales, dtype=tf.float32) + + def _tta( + model: tf.keras.Model, + inputs: tf.Tensor, + spatial_axis: Tuple[int] = SPATIAL_AXIS, + **params, + ): + method_ = partial(method, spatial_axis=spatial_axis, **params) + + shapes = tf.shape(inputs) + sizes = shapes[1:-1] + + logits, maps = _forward(method_, model, inputs, sizes, None, False, resize_method) + + if hflip: + with tf.control_dependencies([logits, maps]): + logits_r, maps_r = _forward(method_, model, inputs, sizes, None, True, resize_method) + logits += logits_r + maps += maps_r + + for idx in tf.range(scales.shape[0]): + scale = scales[idx] + logits_r, maps_r = _forward(method_, model, inputs, sizes, scale, False, resize_method) + logits += logits_r + maps += maps_r + + if hflip: + logits_r, maps_r = _forward(method_, model, inputs, sizes, scale, True, resize_method) + logits += logits_r + maps += maps_r + + repetitions = scales.shape[0] + if hflip: + repetitions *= 2 + + logits /= repetitions + maps /= repetitions + + return logits, maps + + def _forward(method, model, inputs, sizes, scale, hflip, resize_method): + if hflip: + inputs = tf.image.flip_left_right(inputs) + + if scale is not None: + resizes = tf.cast(sizes, tf.float32) + resizes = tf.cast(scale * resizes, tf.int32) + inputs = tf.image.resize(inputs, resizes, method=resize_method) + + logits, maps = method(model, inputs) + + if hflip: + maps = tf.image.flip_left_right(maps) + + maps = tf.image.resize(maps, sizes, method=resize_method) + + return logits, maps + + _tta.__name__ = f'{method.__name__}_tta' + return _tta + +__all__ = [ + "smooth", + "tta", +] diff --git a/src/keras_explainable/utils.py b/src/keras_explainable/utils.py new file mode 100644 index 0000000..a76d681 --- /dev/null +++ b/src/keras_explainable/utils.py @@ -0,0 +1,98 @@ +import io +from math import ceil +from typing import List, Optional, Tuple + +import numpy as np +import tensorflow as tf +from PIL import Image + +# region Generics + +def tolist(item): + if isinstance(item, list): + return item + + if isinstance(item, (tuple, set)): + return list(item) + + return [item] + +# endregion + +# region Visualization + +def get_dims(image): + if hasattr(image, 'shape'): + return image.shape + return (len(image), *get_dims(image[0])) + +def visualize( + images, + title=None, + overlay: Optional[List[np.ndarray]] = None, + overlay_alpha: float = 0.75, + rows: Optional[int] = None, + cols: Optional[int] = None, + figsize: Tuple[float, float] = None, + cmap: str = None, + overlay_cmap: str = None, + to_file: str = None, + to_buffer: io.BytesIO = None, + subplots_ws: float = 0., + subplots_hs: float = 0., +): + import matplotlib.pyplot as plt + + dims = get_dims(images) + rank = len(dims) + + if isinstance(images, tf.Tensor): + images = images.numpy() + + if isinstance(images, (list, tuple)) or rank > 3: + images = images + else: + images = [images] + + if rows is None and cols is None: + cols = min(8, len(images)) + rows = ceil(len(images) / cols) + elif rows is None: + rows = ceil(len(images) / cols) + else: + cols = ceil(len(images) / rows) + + plt.figure(figsize=figsize or (4 * cols, 4 * rows)) + + for ix, image in enumerate(images): + plt.subplot(rows, cols, ix + 1) + + if image is not None: + if isinstance(image, tf.Tensor): + image = image.numpy() + + if len(image.shape) > 2 and image.shape[-1] == 1: + image = image[..., 0] + + plt.imshow(image, cmap=cmap) + + if overlay is not None and len(overlay) > ix and overlay[ix] is not None: + oi = overlay[ix] + if len(oi.shape) > 2 and oi.shape[-1] == 1: + oi = oi[..., 0] + plt.imshow(oi, overlay_cmap, alpha=overlay_alpha) + if title is not None and len(title) > ix: + plt.title(title[ix]) + plt.axis('off') + + plt.tight_layout() + plt.subplots_adjust(wspace=subplots_ws, hspace=subplots_hs) + + if to_buffer: + plt.savefig(to_buffer) + return Image.open(to_buffer) + + if to_file is not None: + plt.savefig(to_file) + +# endregion diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8e4e2f6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +""" + Dummy conftest.py for keras_explainable. + + If you don't know what this is for, just leave it empty. + Read more about conftest.py under: + - https://docs.pytest.org/en/stable/fixture.html + - https://docs.pytest.org/en/stable/writing_plugins.html +""" + +# import pytest diff --git a/tests/unit/engine/explaining_test.py b/tests/unit/engine/explaining_test.py new file mode 100644 index 0000000..5d869f2 --- /dev/null +++ b/tests/unit/engine/explaining_test.py @@ -0,0 +1,166 @@ +from parameterized import parameterized + +import tensorflow as tf +import numpy as np + +import keras_explainable as ke + +TEST_EXPLAIN_SANITY_GRADIENTS_EXCLUDE = ( + ke.methods.gradient.full_gradients, +) + +class ExplainTest(tf.test.TestCase): + BATCH = 2 + SHAPE = [64, 64, 3] + RUN_EAGERLY = False + + def _build_model(self, run_eagerly=False, jit_compile=False): + input_tensor = tf.keras.Input([None, None, 3], name='inputs') + model = tf.keras.applications.ResNet50V2( + weights=None, + input_tensor=input_tensor, + classifier_activation=None, + ) + model.compile( + optimizer='sgd', + loss='sparse_categorical_crossentropy', + metrics=['accuracy'], + run_eagerly=run_eagerly, + jit_compile=jit_compile, + ) + + return model + + def _build_model_with_activations(self, run_eagerly=False, jit_compile=False): + model = self._build_model(run_eagerly, jit_compile) + + return tf.keras.Model( + inputs=model.inputs, + outputs=[model.output, model.get_layer('avg_pool').input] + ) + + @parameterized.expand([(m,) for m in ke.methods.cams.METHODS]) + def test_explain_sanity_cams(self, explaining_method): + model = self._build_model_with_activations() + + x, y = ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + ) + + logits, maps = ke.explain(explaining_method, model, x, y) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) + + @parameterized.expand([ + (False, True), + (True, False), + ]) + def test_explain_cams_jit_compile(self, run_eagerly, jit_compile): + model = self._build_model_with_activations(run_eagerly, jit_compile) + + x, y = ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + ) + + logits, maps = ke.explain(ke.methods.cams.gradcam, model, x, y) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) + + @parameterized.expand([ + (m,) + for m in ke.methods.gradient.METHODS + if m not in TEST_EXPLAIN_SANITY_GRADIENTS_EXCLUDE + ]) + def test_explain_sanity_gradients(self, explaining_method): + model = self._build_model() + + x, y = ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + ) + + logits, maps = ke.explain(explaining_method, model, x, y) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) + + def test_explain_tta_cam(self): + model = self._build_model_with_activations() + + x, y = ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + ) + + explaining_method = ke.methods.meta.tta( + ke.methods.cams.cam, + scales=[0.5], + hflip=True, + ) + logits, maps = ke.explain(explaining_method, model, x, y) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) + + def test_explain_smoothgrad(self): + model = self._build_model() + + x, y = ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + ) + + explaining_method = ke.methods.meta.smooth( + ke.methods.gradient.gradients, + repetitions=3, + noise=0.1, + ) + logits, maps = ke.explain(explaining_method, model, x, y) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) + + def test_explain_sanity_fullgradients(self): + model = self._build_model() + logits = ke.inspection.get_logits_layer(model) + inters, biases = ke.inspection.layers_with_biases(model, exclude=[logits]) + + model_exposed = ke.inspection.expose(model, inters, logits) + + x, y = ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + ) + + logits, maps = ke.explain( + ke.methods.gradient.full_gradients, + model_exposed, + x, + y, + biases=biases, + ) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) diff --git a/tests/unit/methods/meta_test.py b/tests/unit/methods/meta_test.py new file mode 100644 index 0000000..4078c64 --- /dev/null +++ b/tests/unit/methods/meta_test.py @@ -0,0 +1,70 @@ +import numpy as np +import tensorflow as tf + +import keras_explainable as ke + +class MetaTest(tf.test.TestCase): + BATCH = 2 + SHAPE = [64, 64, 3] + RUN_EAGERLY = False + + def _build_model(self): + input_tensor = tf.keras.Input([None, None, 3], name='inputs') + model = tf.keras.applications.ResNet50V2( + weights=None, + input_tensor=input_tensor, + classifier_activation=None, + ) + model.run_eagerly = self.RUN_EAGERLY + + return model + + def _build_model_with_activations(self): + model = self._build_model() + + return tf.keras.Model( + inputs=model.inputs, + outputs=[model.output, model.get_layer('avg_pool').input] + ) + + def test_sanity_tta_cam(self): + model = self._build_model_with_activations() + + x, y = map(tf.convert_to_tensor, ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + )) + + tta = ke.methods.meta.tta( + ke.methods.cams.cam, + scales=[0.5], + hflip=True, + ) + logits, maps = tta(model, x, indices=y) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) + + def test_sanity_smooth_grad(self): + model = self._build_model() + + x, y = map(tf.convert_to_tensor, ( + np.random.rand(self.BATCH, *self.SHAPE), + np.random.randint(10, size=(self.BATCH, 1)) + )) + + smoothgrad = ke.methods.meta.smooth( + ke.methods.gradient.gradients, + repetitions=5, + noise=0.2, + ) + logits, maps = smoothgrad(model, x, y) + + self.assertIsNotNone(logits) + self.assertEqual(logits.shape, (self.BATCH, 1)) + + self.assertIsNotNone(maps) + self.assertEqual(maps.shape, (self.BATCH, *self.SHAPE[:2], 1)) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..3128176 --- /dev/null +++ b/tox.ini @@ -0,0 +1,86 @@ +# Tox configuration file +# Read more under https://tox.wiki/ +# THIS SCRIPT IS SUPPOSED TO BE AN EXAMPLE. MODIFY IT ACCORDING TO YOUR NEEDS! + +[tox] +minversion = 3.24 +envlist = default +isolated_build = True + +[testenv] +description = Invoke pytest to run automated tests +setenv = + TOXINIDIR = {toxinidir} +passenv = + HOME + SETUPTOOLS_* +extras = + testing +commands = + pytest {posargs} + +# # To run `tox -e lint` you need to make sure you have a +# # `.pre-commit-config.yaml` file. See https://pre-commit.com +# [testenv:lint] +# description = Perform static analysis and style checks +# skip_install = True +# deps = pre-commit +# passenv = +# HOMEPATH +# PROGRAMDATA +# SETUPTOOLS_* +# commands = +# pre-commit run --all-files {posargs:--show-diff-on-failure} + +[testenv:{build,clean}] +description = + build: Build the package in isolation according to PEP517, see https://github.com/pypa/build + clean: Remove old distribution files and temporary build artifacts (./build and ./dist) +# https://setuptools.pypa.io/en/stable/build_meta.html#how-to-use-it +skip_install = True +changedir = {toxinidir} +deps = + build: build[virtualenv] +passenv = + SETUPTOOLS_* +commands = + clean: python -c 'import shutil; [shutil.rmtree(p, True) for p in ("build", "dist", "docs/_build")]' + clean: python -c 'import pathlib, shutil; [shutil.rmtree(p, True) for p in pathlib.Path("src").glob("*.egg-info")]' + build: python -m build {posargs} + +[testenv:{docs,doctests,linkcheck}] +description = + docs: Invoke sphinx-build to build the docs + doctests: Invoke sphinx-build to run doctests + linkcheck: Check for broken links in the documentation +passenv = + SETUPTOOLS_* +setenv = + DOCSDIR = {toxinidir}/docs + BUILDDIR = {toxinidir}/docs/_build + docs: BUILD = html + doctests: BUILD = doctest + linkcheck: BUILD = linkcheck +deps = + -r {toxinidir}/docs/requirements.txt + # ^ requirements.txt shared with Read The Docs +commands = + sphinx-build --color -b {env:BUILD} -d "{env:BUILDDIR}/doctrees" "{env:DOCSDIR}" "{env:BUILDDIR}/{env:BUILD}" {posargs} + +[testenv:publish] +description = + Publish the package you have been developing to a package index server. + By default, it uses testpypi. If you really want to publish your package + to be publicly accessible in PyPI, use the `-- --repository pypi` option. +skip_install = True +changedir = {toxinidir} +passenv = + # See: https://twine.readthedocs.io/en/latest/ + TWINE_USERNAME + TWINE_PASSWORD + TWINE_REPOSITORY + TWINE_REPOSITORY_URL +deps = twine +commands = + python -m twine check dist/* + python -m twine upload {posargs:--repository {env:TWINE_REPOSITORY:testpypi}} dist/*