diff --git a/.pytest_cache/v/cache/lastfailed b/.pytest_cache/v/cache/lastfailed deleted file mode 100644 index 9e26dfe..0000000 --- a/.pytest_cache/v/cache/lastfailed +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/.pytest_cache/v/cache/nodeids b/.pytest_cache/v/cache/nodeids deleted file mode 100644 index 2e606dd..0000000 --- a/.pytest_cache/v/cache/nodeids +++ /dev/null @@ -1,26 +0,0 @@ -[ - "src/importnb/tests/test_importnb.ipynb::test_single_file_with_context", - "src/importnb/tests/test_importnb.ipynb::test_from_filename", - "src/importnb/tests/test_importnb.ipynb::test_with_doctest", - "src/importnb/tests/test_importnb.ipynb::test_from_filename_main", - "src/importnb/tests/test_importnb.ipynb::test_parameterize", - "src/importnb/tests/test_importnb.ipynb::test_python_file", - "src/importnb/tests/test_importnb.ipynb::test_single_file_with_capture", - "src/importnb/tests/test_importnb.ipynb::test_capturer", - "src/importnb/tests/test_importnb.ipynb::test_single_file_with_lazy", - "src/importnb/tests/test_importnb.ipynb::test_single_file_without_context", - "src/importnb/tests/test_importnb.ipynb::test_single_file_relative", - "src/importnb/tests/test_importnb.ipynb::test_single_with_extension", - "src/importnb/tests/test_importnb.ipynb::test_package", - "src/importnb/tests/test_importnb.ipynb::test_package_failure", - "src/importnb/tests/test_importnb.ipynb::test_package_failure_partial", - "src/importnb/tests/test_module.ipynb::testmod", - "src/importnb/tests/test_module.ipynb::test_all", - "src/importnb/tests/test_unittests.ipynb::TestExtension::test_failure", - "src/importnb/tests/test_unittests.ipynb::TestExtension::test_import", - "src/importnb/tests/test_unittests.ipynb::TestContext::test_import", - "src/importnb/tests/test_unittests.ipynb::TestContext::test_reload_with_context", - "src/importnb/tests/test_unittests.ipynb::TestRemote::test_imports", - "src/importnb/tests/test_unittests.ipynb::TestPartial::test_exception", - "src/importnb/tests/test_unittests.ipynb::TestPartial::test_traceback" -] \ No newline at end of file diff --git a/classes_importnb.png b/classes_importnb.png deleted file mode 100644 index 96cd3ab..0000000 Binary files a/classes_importnb.png and /dev/null differ diff --git a/packages_importnb.png b/packages_importnb.png deleted file mode 100644 index 9c27c6a..0000000 Binary files a/packages_importnb.png and /dev/null differ diff --git a/readme.ipynb b/readme.ipynb index a76262f..ee927f8 100644 --- a/readme.ipynb +++ b/readme.ipynb @@ -409,7 +409,7 @@ "output_type": "stream", "text": [ "[NbConvertApp] Converting notebook readme.ipynb to markdown\n", - "[NbConvertApp] Writing 6469 bytes to readme.md\n" + "[NbConvertApp] Writing 7022 bytes to readme.md\n" ] } ], diff --git a/readme.md b/readme.md index 4d457ec..2eb7515 100644 --- a/readme.md +++ b/readme.md @@ -224,7 +224,18 @@ For example, create a file called `tricks.yaml` containing ``` test_import (src.importnb.tests.test_unittests.TestContext) ... ok - test_reload_with_context (src.importnb.tests.test_unittests.TestContext) ... + test_reload_with_context (src.importnb.tests.test_unittests.TestContext) ... ok + test_failure (src.importnb.tests.test_unittests.TestExtension) ... expected failure + test_import (src.importnb.tests.test_unittests.TestExtension) ... ok + test_exception (src.importnb.tests.test_unittests.TestPartial) ... ok + test_traceback (src.importnb.tests.test_unittests.TestPartial) ... ok + test_imports (src.importnb.tests.test_unittests.TestRemote) ... skipped 'requires IP' + + ---------------------------------------------------------------------- + Ran 7 tests in 2.018s + + OK (skipped=1, expected failures=1) + ```python diff --git a/readme_files/readme_33_1.png b/readme_files/readme_33_1.png deleted file mode 100644 index 944a757..0000000 Binary files a/readme_files/readme_33_1.png and /dev/null differ diff --git a/readme_files/readme_33_2.png b/readme_files/readme_33_2.png deleted file mode 100644 index 9975ea3..0000000 Binary files a/readme_files/readme_33_2.png and /dev/null differ diff --git a/src/importnb/__init__.py b/src/importnb/__init__.py index 9f0a537..b9a46bd 100644 --- a/src/importnb/__init__.py +++ b/src/importnb/__init__.py @@ -11,8 +11,6 @@ unload_ipython_extension, reload, Lazy, - export, - __IPYTHON__, ) from .parameterize import Parameterize from .nbtest import NotebookTest, testmod @@ -25,8 +23,6 @@ unload_ipython_extension, reload, Lazy, - export, - __IPYTHON__, ) from parameterize import Parameterize from nbtest import NotebookTest, testmod \ No newline at end of file diff --git a/src/importnb/capture.py b/src/importnb/capture.py index 1e90a6d..60e7b08 100644 --- a/src/importnb/capture.py +++ b/src/importnb/capture.py @@ -6,15 +6,11 @@ try: - from .compile import export, __IPYTHON__, export -except: - from compile import export, __IPYTHON__, export -__all__ = "capture_output", - - -if __IPYTHON__: from IPython.utils.capture import capture_output -else: + from IPython import get_ipython + + assert get_ipython(), """There is no interactive shell""" +except: from contextlib import redirect_stdout, ExitStack from io import StringIO @@ -73,5 +69,9 @@ def stderr(self): if __name__ == "__main__": + try: + from .utils.export import export + except: + from utils.export import export export("capture.ipynb", "../capture.py") __import__("doctest").testmod() diff --git a/src/importnb/compile.py b/src/importnb/compile.py deleted file mode 100644 index 9ff0b40..0000000 --- a/src/importnb/compile.py +++ /dev/null @@ -1,141 +0,0 @@ - -# coding: utf-8 - -"""# The `compile` module - -...provides compatibility for Python and IPython through [`compile_python`](compile_python.ipynb) and [`compile_ipython`](compile_ipython.ipynb), respectively. - - >>> from importnb.compile import __IPYTHON__, Compiler, PythonExporter -""" - - -import ast, sys -from pathlib import Path - -__file__ = globals().get("__file__", "compile.ipynb") -__nb__ = __file__.replace("src/importnb", "src/importnb") - -__IPYTHON__ = False - -try: - from IPython import get_ipython - - if not get_ipython(): - ... - else: - __IPYTHON__ = True -except: - ... - -if __IPYTHON__: - try: - from .compile_ipython import Compiler, PythonExporter - except: - from compile_ipython import Compiler, PythonExporter -else: - try: - from .compile_python import Compiler, PythonExporter - except: - from compile_python import Compiler, PythonExporter - - -class ImportNbStyleExporter(PythonExporter): - PythonExporter.exclude_input_prompt = True - - def from_notebook_node(self, nb, resources=None, **kw): - code, resources = super().from_notebook_node(nb, resources=resources, **kw) - try: - from black import format_str - except: - format_str = lambda x, i: x - return format_str(code, 100), resources - - -def export(file, to=None): - from pathlib import Path - - exporter = ImportNbStyleExporter() - code = exporter.from_filename(file)[0] - if to: - Path(to).with_suffix(exporter.file_extension).write_text(code) - return code - - -class Code(PythonExporter, Compiler): - """An exporter than returns transforms a NotebookNode through the InputSplitter. - - >>> assert type(Code().from_filename(Path(__nb__).with_suffix('.ipynb'))) is __import__('nbformat').NotebookNode""" - - def __init__(self, filename="", name="__main__"): - PythonExporter.__init__(self) - Compiler.__init__(self) - self.filename = filename - self.name = name - - def from_file(Code, file_stream, resources=None, **dict): - for str in ("name", "filename"): - setattr(Code, str, dict.pop(str, getattr(Code, str))) - return super().from_file(file_stream, resources, **dict) - - def from_filename(Code, filename, resources=None, **dict): - Code.filename, Code.name = filename, Path(filename).stem - return super().from_filename(filename, resources, **dict) - - def from_notebook_node(Code, nb, resources=None, **dict): - for index, cell in enumerate(nb["cells"]): - if cell["cell_type"] == "code": - if isinstance(cell["source"], list): - cell["source"] = "".join(cell["source"]) - cell["source"] = Code.from_code_cell(cell, **dict) - return nb - - def from_code_cell(Code, cell, **dict): - return Code.transform(cell["source"]) - - -class AST(Code): - """An exporter than returns parsed ast. - - >>> assert type(AST().from_filename(Path(__nb__).with_suffix('.ipynb'))) is ast.Module""" - - def from_notebook_node(AST, nb, resource=None, **dict): - module = AST.ast_transform( - ast.fix_missing_locations( - ast.Module( - body=sum( - ( - AST.ast_parse( - AST.from_code_cell(cell, **dict), - lineno=cell["metadata"].get("lineno", 1), - ).body - for cell in super().from_notebook_node(nb, resource, **dict)["cells"] - if cell["cell_type"] == "code" - ), - [], - ) - ) - ) - ) - - # The module ast node has a docstring parameter. - # https://docs.python.org/3.7/whatsnew/3.7.html#changes-in-the-python-api - - return module - - -class Compile(AST): - """An exporter that returns compiled and cached bytecode. - - >>> assert Compile().from_filename(Path(__nb__).with_suffix('.ipynb'))""" - - def from_notebook_node(Compile, nb, resources: dict = None, **dict): - return Compile.compile(super().from_notebook_node(nb, resources, **dict)) - - -if __name__ == "__main__": - export("compile.ipynb", "../compile.py") - try: - from . import compile - except: - import compile - __import__("doctest").testmod(compile, verbose=2) diff --git a/src/importnb/compile_ipython.py b/src/importnb/compile_ipython.py deleted file mode 100644 index 1c4a287..0000000 --- a/src/importnb/compile_ipython.py +++ /dev/null @@ -1,66 +0,0 @@ - -# coding: utf-8 - -"""# The IPython compiler -""" - - -try: - from .decoder import load -except: - from decoder import load - -from nbconvert.exporters.python import PythonExporter as _PythonExporter -from nbconvert.exporters.notebook import NotebookExporter -from nbformat import from_dict -from IPython.core.compilerop import CachingCompiler -from IPython.core.inputsplitter import IPythonInputSplitter -import ast - - -class IpythonCompiler(CachingCompiler): - """{Shell} provides the IPython machinery to objects.""" - transform = staticmethod(IPythonInputSplitter().transform_cell) - - @property - def ip(Compiler): - """The current interactive shell""" - from IPython import get_ipython - from IPython.core.interactiveshell import InteractiveShell - - return get_ipython() or InteractiveShell() - - def ast_transform(Compiler, node): - for visitor in Compiler.ip.ast_transformers: - node = visitor.visit(node) - return node - - def compile(Compiler, ast): - """Compile AST to bytecode using the an IPython compiler.""" - return (Compiler.ip and Compiler.ip.compile or CachingCompiler())( - ast, Compiler.filename, "exec" - ) - - def ast_parse(Compiler, source, filename="", symbol="exec", lineno=0): - return ast.increment_lineno(super().ast_parse(source, Compiler.filename, "exec"), lineno) - - -Compiler = IpythonCompiler - - -class IpythonPythonExporter(_PythonExporter): - - def from_file(self, file_stream, resources=None, **kw): - return self.from_notebook_node(from_dict(load(file_stream)), resources, **kw) - - -PythonExporter = IpythonPythonExporter - - -if __name__ == "__main__": - try: - from .compile import export - except: - from compile import export - export("compile_ipython.ipynb", "../compile_ipython.py") - __import__("doctest").testmod() diff --git a/src/importnb/compile_python.py b/src/importnb/compile_python.py deleted file mode 100644 index c9d075e..0000000 --- a/src/importnb/compile_python.py +++ /dev/null @@ -1,76 +0,0 @@ - -# coding: utf-8 - -"""# The Python compile module -""" - - -import ast, sys - -try: - from .decoder import load -except: - from decoder import load -from pathlib import Path -from textwrap import dedent -from codeop import Compile - - -class PythonCompiler(Compile): - """{Shell} provides the IPython machinery to objects.""" - - def ast_transform(Compiler, node): - return node - - @property - def transform(Compiler): - return dedent - - def compile(Compiler, ast): - return compile(ast, Compiler.filename, "exec") - - def ast_parse(Compiler, source, filename="", symbol="exec", lineno=0): - return ast.increment_lineno(ast.parse(source, Compiler.filename, "exec"), lineno) - - -Compiler = PythonCompiler - - -class PythonNotebookExporter: - - def from_file(self, file_stream, resources=None, **kw): - return self.from_notebook_node(load(file_stream), resources, **kw) - - def from_filename(self, filename, resources=None, **dict): - with open(filename, "r") as file_stream: - return self.from_file(file_stream, resources, **dict) - - def from_notebook_node(self, nb, resources=None, **dict): - return nb, resources - - -NotebookExporter = PythonNotebookExporter - - -class PythonPythonExporter(NotebookExporter): - - def from_notebook_node(self, nb, resources=None, **dict): - nb, resources = super().from_notebook_node(nb, resources, **dict) - for i, cell in enumerate(nb["cells"]): - if isinstance(cell["source"], list): - nb["cells"][i]["source"] = "".join(cell["source"]) - return ("\n" * 2).join( - dedent(cell["source"]) for cell in nb["cells"] if cell["cell_type"] == "code" - ), resources - - -PythonExporter = PythonPythonExporter - - -if __name__ == "__main__": - try: - from .compile import export - except: - from compile import export - export("compile_python.ipynb", "../compile_python.py") - __import__("doctest").testmod() diff --git a/src/importnb/decoder.py b/src/importnb/decoder.py index b018898..51b9a1a 100644 --- a/src/importnb/decoder.py +++ b/src/importnb/decoder.py @@ -12,8 +12,19 @@ from json import load as _load, loads as _loads from functools import partial +try: + from IPython.core.inputsplitter import IPythonInputSplitter -class LineNoDecoder(JSONDecoder): + dedent = IPythonInputSplitter().transform_cell +except: + from textwrap import dedent + + +def identity(*x): + return x[0] + + +class LineNumberDecoder(JSONDecoder): """A JSON Decoder to return a NotebookNode with lines numbers in the metadata.""" def __init__( @@ -36,10 +47,10 @@ def __init__( strict=strict, object_pairs_hook=object_pairs_hook, ) - self.parse_object = self.object + self.parse_object = self._parse_object self.scan_once = py_make_scanner(self) - def object( + def _parse_object( self, s_and_end, strict, @@ -53,7 +64,6 @@ def object( object, next = JSONObject( s_and_end, strict, scan_once, object_hook, object_pairs_hook, memo=memo, _w=_w, _ws=_ws ) - if "cell_type" in object: object["metadata"].update( {"lineno": len(s_and_end[0][:next].rsplit('"source":', 1)[0].splitlines())} @@ -68,6 +78,7 @@ def object( for key in ("source", "text"): if key in object: object[key] = "".join(object[key]) + return object, next @@ -90,15 +101,44 @@ def codify_markdown_list(str): return list(map("{}\n".format, codify_markdown_string("".join(str)).splitlines())) -load = partial(_load, cls=LineNoDecoder) -loads = partial(_loads, cls=LineNoDecoder) +load = partial(_load, cls=LineNumberDecoder) +loads = partial(_loads, cls=LineNumberDecoder) + + +def transform_cells(object, transform=dedent): + for cell in object["cells"]: + if "source" in cell: + cell["source"] = transform("".join(cell["source"])) + return object + + +def ast_from_cells(object, transform=identity): + import ast + + module = ast.Module(body=[]) + for cell in object["cells"]: + module.body.extend( + ast.fix_missing_locations( + ast.increment_lineno( + ast.parse("".join(cell["source"])), cell["metadata"].get("lineno", 1) + ) + ).body + ) + return module + + +def loads_ast(object, loads=loads, transform=dedent, ast_transform=identity): + if isinstance(object, str): + object = loads(object) + object = transform_cells(object, transform) + return ast_from_cells(object, ast_transform) if __name__ == "__main__": try: - from .compile import export + from .utils.export import export except: - from compile import export + from utils.export import export export("decoder.ipynb", "../decoder.py") __import__("doctest").testmod() diff --git a/src/importnb/loader.py b/src/importnb/loader.py index 6835455..cfa5a9f 100644 --- a/src/importnb/loader.py +++ b/src/importnb/loader.py @@ -27,12 +27,11 @@ try: - from .compile import __IPYTHON__, export, Compile, AST from .capture import capture_output + from .decoder import loads_ast, identity, loads, dedent except: - from compile import __IPYTHON__, export, Compile, AST from capture import capture_output - + from decoder import loads_ast, identity, loads, dedent import inspect, sys from importlib.machinery import SourceFileLoader @@ -127,6 +126,11 @@ class Notebook(SourceFileLoader, capture_output): """A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.""" EXTENSION_SUFFIXES = ".ipynb", + _compile = staticmethod(compile) + _loads = staticmethod(loads) + _transform = staticmethod(dedent) + _ast_transform = staticmethod(identity) + def __init__( self, fullname=None, @@ -172,9 +176,17 @@ def exec_module(self, module): def __exit__(self, *excepts): remove_one_path_hook(type(self)), super().__exit__(*excepts) - def source_to_code(Notebook, data, path): - with StringIO(data.decode("utf-8")) as stream: - return Compile().from_file(stream, filename=Notebook.path, name=Notebook.name) + def source_to_code(self, data, path): + return self._compile( + loads_ast( + data.decode("utf-8"), + loads=self._loads, + transform=self._transform, + ast_transform=self._ast_transform, + ), + path or "", + "exec", + ) def from_filename(self, file): """Load a python module or notebook from a file location. @@ -269,5 +281,9 @@ def unload_ipython_extension(ip=None): if __name__ == "__main__": + try: + from utils.export import export + except: + from .utils.export import export export("loader.ipynb", "../loader.py") __import__("doctest").testmod(Notebook()("loader.ipynb")) diff --git a/src/importnb/nbtest.py b/src/importnb/nbtest.py index 1560deb..52e5bce 100644 --- a/src/importnb/nbtest.py +++ b/src/importnb/nbtest.py @@ -11,9 +11,9 @@ try: - from .loader import Notebook, export + from .loader import Notebook except: - from loader import Notebook, export + from loader import Notebook from unittest import TestProgram, TestCase from doctest import DocTestSuite diff --git a/src/importnb/notebooks/capture.ipynb b/src/importnb/notebooks/capture.ipynb index c1c5656..1ad5783 100644 --- a/src/importnb/notebooks/capture.ipynb +++ b/src/importnb/notebooks/capture.ipynb @@ -14,21 +14,10 @@ "outputs": [], "source": [ "try:\n", - " from .compile import export, __IPYTHON__, export\n", - "except:\n", - " from compile import export, __IPYTHON__, export\n", - "__all__ = 'capture_output'," - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "if __IPYTHON__:\n", " from IPython.utils.capture import capture_output\n", - "else:\n", + " from IPython import get_ipython\n", + " assert get_ipython(), \"\"\"There is no interactive shell\"\"\"\n", + "except:\n", " from contextlib import redirect_stdout, ExitStack\n", " from io import StringIO\n", " try:\n", @@ -90,6 +79,8 @@ "outputs": [], "source": [ " if __name__ == '__main__':\n", + " try: from .utils.export import export\n", + " except: from utils.export import export\n", " export('capture.ipynb', '../capture.py')\n", " __import__('doctest').testmod()" ] diff --git a/src/importnb/notebooks/compile.ipynb b/src/importnb/notebooks/compile.ipynb deleted file mode 100644 index 6b13ac2..0000000 --- a/src/importnb/notebooks/compile.ipynb +++ /dev/null @@ -1,242 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The `compile` module\n", - "\n", - "...provides compatibility for Python and IPython through [`compile_python`](compile_python.ipynb) and [`compile_ipython`](compile_ipython.ipynb), respectively. \n", - "\n", - " >>> from importnb.compile import __IPYTHON__, Compiler, PythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " import ast, sys\n", - " from pathlib import Path\n", - " \n", - " __file__ = globals().get('__file__', 'compile.ipynb')\n", - " __nb__ = __file__.replace('src/importnb', 'src/importnb')\n", - "\n", - " __IPYTHON__ = False\n", - "\n", - " try:\n", - " from IPython import get_ipython\n", - " if not get_ipython(): ...\n", - " else: __IPYTHON__ = True\n", - " except: ...\n", - "\n", - " if __IPYTHON__:\n", - " try:\n", - " from .compile_ipython import Compiler, PythonExporter\n", - " except:\n", - " from compile_ipython import Compiler, PythonExporter\n", - " else:\n", - " try:\n", - " from .compile_python import Compiler, PythonExporter\n", - " except:\n", - " from compile_python import Compiler, PythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " class ImportNbStyleExporter(PythonExporter):\n", - " PythonExporter.exclude_input_prompt = True\n", - " def from_notebook_node(self, nb, resources=None, **kw):\n", - " code, resources = super().from_notebook_node(nb, resources=resources, **kw)\n", - " try:\n", - " from black import format_str\n", - " except:\n", - " format_str = lambda x, i: x\n", - " return format_str(code, 100), resources" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - " def export(file, to=None): \n", - " from pathlib import Path\n", - " exporter = ImportNbStyleExporter()\n", - " code = exporter.from_filename(file)[0]\n", - " if to: Path(to).with_suffix(exporter.file_extension).write_text(code)\n", - " return code " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - " class Code(PythonExporter, Compiler):\n", - " \"\"\"An exporter than returns transforms a NotebookNode through the InputSplitter.\n", - " \n", - " >>> assert type(Code().from_filename(Path(__nb__).with_suffix('.ipynb'))) is __import__('nbformat').NotebookNode\"\"\"\n", - " def __init__(\n", - " self, \n", - " filename='', name='__main__'\n", - " ): \n", - " PythonExporter.__init__(self) \n", - " Compiler.__init__(self)\n", - " self.filename = filename\n", - " self.name = name\n", - " \n", - " def from_file(Code, file_stream, resources=None, **dict): \n", - " for str in ('name', 'filename'): setattr(Code, str, dict.pop(str, getattr(Code, str)))\n", - " return super().from_file(file_stream, resources, **dict)\n", - " \n", - " def from_filename(Code, filename, resources=None, **dict):\n", - " Code.filename, Code.name = filename, Path(filename).stem\n", - " return super().from_filename(filename, resources, **dict)\n", - "\n", - " def from_notebook_node(Code, nb, resources=None, **dict): \n", - " for index, cell in enumerate(nb['cells']):\n", - " if cell['cell_type'] == 'code':\n", - " if isinstance(cell['source'], list):\n", - " cell['source'] = ''.join(cell['source'])\n", - " cell['source'] = Code.from_code_cell(cell, **dict)\n", - " return nb\n", - " \n", - " def from_code_cell(Code, cell, **dict): \n", - " return Code.transform(cell['source'])" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - " class AST(Code):\n", - " \"\"\"An exporter than returns parsed ast.\n", - " \n", - " >>> assert type(AST().from_filename(Path(__nb__).with_suffix('.ipynb'))) is ast.Module\"\"\"\n", - " def from_notebook_node(AST, nb, resource=None, **dict): \n", - " module = AST.ast_transform(ast.fix_missing_locations(ast.Module(body=sum((\n", - " AST.ast_parse(\n", - " AST.from_code_cell(cell, **dict), \n", - " lineno=cell['metadata'].get('lineno', 1)\n", - " ).body \n", - " for cell in super().from_notebook_node(nb, resource, **dict)['cells'] \n", - " if cell['cell_type']=='code'\n", - " ), []))))\n", - "\n", - " # The module ast node has a docstring parameter.\n", - " # https://docs.python.org/3.7/whatsnew/3.7.html#changes-in-the-python-api\n", - " \n", - " return module" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - " class Compile(AST):\n", - " \"\"\"An exporter that returns compiled and cached bytecode.\n", - " \n", - " >>> assert Compile().from_filename(Path(__nb__).with_suffix('.ipynb'))\"\"\" \n", - " def from_notebook_node(Compile, nb, resources: dict=None, **dict):\n", - " return Compile.compile(super().from_notebook_node(nb, resources, **dict))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trying:\n", - " from importnb.compile import __IPYTHON__, Compiler, PythonExporter\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert type(AST().from_filename(Path(__nb__).with_suffix('.ipynb'))) is ast.Module\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert type(Code().from_filename(Path(__nb__).with_suffix('.ipynb'))) is __import__('nbformat').NotebookNode\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert Compile().from_filename(Path(__nb__).with_suffix('.ipynb'))\n", - "Expecting nothing\n", - "ok\n", - "10 items had no tests:\n", - " compile.AST.from_notebook_node\n", - " compile.Code.__init__\n", - " compile.Code.from_code_cell\n", - " compile.Code.from_file\n", - " compile.Code.from_filename\n", - " compile.Code.from_notebook_node\n", - " compile.Compile.from_notebook_node\n", - " compile.ImportNbStyleExporter\n", - " compile.ImportNbStyleExporter.from_notebook_node\n", - " compile.export\n", - "4 items passed all tests:\n", - " 1 tests in compile\n", - " 1 tests in compile.AST\n", - " 1 tests in compile.Code\n", - " 1 tests in compile.Compile\n", - "4 tests in 14 items.\n", - "4 passed and 0 failed.\n", - "Test passed.\n" - ] - } - ], - "source": [ - " if __name__ == '__main__':\n", - " export('compile.ipynb', '../compile.py')\n", - " try: from . import compile\n", - " except: import compile\n", - " __import__('doctest').testmod(compile, verbose=2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/importnb/notebooks/compile_ipython.ipynb b/src/importnb/notebooks/compile_ipython.ipynb deleted file mode 100644 index 691ef60..0000000 --- a/src/importnb/notebooks/compile_ipython.ipynb +++ /dev/null @@ -1,166 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The IPython compiler" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " try:\n", - " from .decoder import load\n", - " except:\n", - " from decoder import load\n", - "\n", - " from nbconvert.exporters.python import PythonExporter as _PythonExporter\n", - " from nbconvert.exporters.notebook import NotebookExporter\n", - " from nbformat import from_dict\n", - " from IPython.core.compilerop import CachingCompiler\n", - " from IPython.core.inputsplitter import IPythonInputSplitter\n", - " import ast\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " class IpythonCompiler(CachingCompiler):\n", - " \"\"\"{Shell} provides the IPython machinery to objects.\"\"\"\n", - " transform = staticmethod(IPythonInputSplitter().transform_cell)\n", - " @property\n", - " def ip(Compiler): \n", - " \"\"\"The current interactive shell\"\"\"\n", - " from IPython import get_ipython\n", - " from IPython.core.interactiveshell import InteractiveShell\n", - " return get_ipython() or InteractiveShell()\n", - " \n", - " def ast_transform(Compiler, node):\n", - " for visitor in Compiler.ip.ast_transformers: \n", - " node = visitor.visit(node)\n", - " return node\n", - " \n", - " def compile(Compiler, ast): \n", - " \"\"\"Compile AST to bytecode using the an IPython compiler.\"\"\"\n", - " return (Compiler.ip and Compiler.ip.compile or CachingCompiler())(ast, Compiler.filename, 'exec')\n", - " \n", - " def ast_parse(Compiler, source, filename='', symbol='exec', lineno=0): \n", - " return ast.increment_lineno(super().ast_parse(source, Compiler.filename, 'exec'), lineno)\n", - " \n", - " Compiler = IpythonCompiler" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - " class IpythonPythonExporter(_PythonExporter):\n", - " def from_file(self, file_stream, resources=None, **kw):\n", - " return self.from_notebook_node(from_dict(load(file_stream)), resources, **kw)\n", - " PythonExporter = IpythonPythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - " if __name__ == '__main__':\n", - " try: from .compile import export\n", - " except: from compile import export\n", - " export('compile_ipython.ipynb', '../compile_ipython.py')\n", - " __import__('doctest').testmod()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - }, - "toc": { - "colors": { - "hover_highlight": "#DAA520", - "running_highlight": "#FF0000", - "selected_highlight": "#FFD700" - }, - "moveMenuLeft": true, - "nav_menu": { - "height": "29px", - "width": "252px" - }, - "navigate_menu": true, - "number_sections": true, - "sideBar": true, - "threshold": 4, - "toc_cell": false, - "toc_section_display": "block", - "toc_window_display": false, - "widenNotebook": false - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/importnb/notebooks/compile_python.ipynb b/src/importnb/notebooks/compile_python.ipynb deleted file mode 100644 index 7db20ef..0000000 --- a/src/importnb/notebooks/compile_python.ipynb +++ /dev/null @@ -1,170 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The Python compile module" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " import ast, sys\n", - " try:\n", - " from .decoder import load\n", - " except:\n", - " from decoder import load\n", - " from pathlib import Path\n", - " from textwrap import dedent\n", - " from codeop import Compile" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " class PythonCompiler(Compile):\n", - " \"\"\"{Shell} provides the IPython machinery to objects.\"\"\"\n", - " def ast_transform(Compiler, node): return node\n", - " \n", - " @property\n", - " def transform(Compiler): return dedent\n", - "\n", - " def compile(Compiler, ast): return compile(ast, Compiler.filename, 'exec')\n", - " \n", - " def ast_parse(Compiler, source, filename='', symbol='exec', lineno=0): \n", - " return ast.increment_lineno(ast.parse(source, Compiler.filename, 'exec'), lineno)\n", - " \n", - " Compiler = PythonCompiler" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - " class PythonNotebookExporter:\n", - " def from_file(self, file_stream, resources=None, **kw):\n", - " return self.from_notebook_node(load(file_stream), resources, **kw) \n", - " \n", - " def from_filename(self, filename, resources=None, **dict):\n", - " with open(filename, 'r') as file_stream:\n", - " return self.from_file(file_stream, resources, **dict)\n", - " \n", - " def from_notebook_node(self, nb, resources=None, **dict): \n", - " return nb, resources\n", - " NotebookExporter = PythonNotebookExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - " class PythonPythonExporter(NotebookExporter):\n", - " def from_notebook_node(self, nb, resources=None, **dict):\n", - " nb, resources = super().from_notebook_node(nb, resources, **dict)\n", - " for i, cell in enumerate(nb['cells']):\n", - " if isinstance(cell['source'], list):\n", - " nb['cells'][i]['source'] = ''.join(cell['source'])\n", - " return (\"\\n\"*2).join(\n", - " dedent(cell['source']) for cell in nb['cells'] if cell['cell_type'] == 'code'\n", - " ), resources\n", - " PythonExporter = PythonPythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - " if __name__ == '__main__':\n", - " try: from .compile import export\n", - " except: from compile import export\n", - " export('compile_python.ipynb', '../compile_python.py')\n", - " __import__('doctest').testmod()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - }, - "toc": { - "colors": { - "hover_highlight": "#DAA520", - "running_highlight": "#FF0000", - "selected_highlight": "#FFD700" - }, - "moveMenuLeft": true, - "nav_menu": { - "height": "29px", - "width": "252px" - }, - "navigate_menu": true, - "number_sections": true, - "sideBar": true, - "threshold": 4, - "toc_cell": false, - "toc_section_display": "block", - "toc_window_display": false, - "widenNotebook": false - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/importnb/notebooks/decoder.ipynb b/src/importnb/notebooks/decoder.ipynb index d9f0273..638e49d 100644 --- a/src/importnb/notebooks/decoder.ipynb +++ b/src/importnb/notebooks/decoder.ipynb @@ -18,7 +18,13 @@ " from functools import singledispatch\n", " from json.decoder import JSONObject, JSONDecoder, WHITESPACE, WHITESPACE_STR\n", " from json import load as _load, loads as _loads\n", - " from functools import partial" + " from functools import partial\n", + " \n", + " try:\n", + " from IPython.core.inputsplitter import IPythonInputSplitter\n", + " dedent = IPythonInputSplitter().transform_cell\n", + " except:\n", + " from textwrap import dedent" ] }, { @@ -27,22 +33,30 @@ "metadata": {}, "outputs": [], "source": [ - " class LineNoDecoder(JSONDecoder):\n", + " def identity(*x): return x[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + " class LineNumberDecoder(JSONDecoder):\n", " \"\"\"A JSON Decoder to return a NotebookNode with lines numbers in the metadata.\"\"\"\n", " def __init__(self, *, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None):\n", - " from json.scanner import py_make_scanner \n", + " from json.scanner import py_make_scanner\n", " super().__init__(object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, parse_constant=parse_constant, strict=strict, \n", " object_pairs_hook=object_pairs_hook)\n", - " self.parse_object = self.object\n", + " self.parse_object = self._parse_object\n", " self.scan_once = py_make_scanner(self)\n", " \n", - " def object(\n", + " def _parse_object(\n", " self, \n", " s_and_end, \n", " strict, scan_once, object_hook, object_pairs_hook, memo=None, _w=WHITESPACE.match, _ws=WHITESPACE_STR\n", " ) -> (dict, int):\n", " object, next = JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, memo=memo, _w=_w, _ws=_ws)\n", - "\n", " if 'cell_type' in object: \n", " object['metadata'].update({\n", " 'lineno': len(s_and_end[0][:next].rsplit('\"source\":', 1)[0].splitlines())\n", @@ -56,12 +70,13 @@ " \n", " for key in ('source', 'text'): \n", " if key in object: object[key] = ''.join(object[key])\n", + " \n", " return object, next" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -83,21 +98,60 @@ " )\n", "\n", " \n", - " load = partial(_load, cls=LineNoDecoder)\n", - " loads = partial(_loads, cls=LineNoDecoder)" + " load = partial(_load, cls=LineNumberDecoder)\n", + " loads = partial(_loads, cls=LineNumberDecoder)" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + " def transform_cells(object, transform=dedent):\n", + " for cell in object['cells']:\n", + " if 'source' in cell:\n", + " cell['source'] = transform(''.join(cell['source']))\n", + " return object\n", + " \n", + " def ast_from_cells(object, transform=identity):\n", + " import ast\n", + " module = ast.Module(body=[])\n", + " for cell in object['cells']:\n", + " module.body.extend(\n", + " ast.fix_missing_locations(\n", + " ast.increment_lineno(\n", + " ast.parse(''.join(cell['source'])), \n", + " cell['metadata'].get('lineno', 1)\n", + " )\n", + " ).body)\n", + " return module" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + " def loads_ast(object, loads=loads, transform=dedent, ast_transform=identity):\n", + " if isinstance(object, str):\n", + " object = loads(object)\n", + " object = transform_cells(object, transform)\n", + " return ast_from_cells(object, ast_transform)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, "metadata": { "scrolled": false }, "outputs": [], "source": [ " if __name__ == '__main__':\n", - " try: from .compile import export\n", - " except: from compile import export\n", + " try: from .utils.export import export\n", + " except: from utils.export import export \n", " export('decoder.ipynb', '../decoder.py')\n", "\n", " __import__('doctest').testmod()" diff --git a/src/importnb/notebooks/loader.ipynb b/src/importnb/notebooks/loader.ipynb index 4021bdd..ab59d7c 100644 --- a/src/importnb/notebooks/loader.ipynb +++ b/src/importnb/notebooks/loader.ipynb @@ -35,19 +35,12 @@ "outputs": [], "source": [ " try:\n", - " from .compile import __IPYTHON__, export, Compile, AST\n", " from .capture import capture_output\n", + " from .decoder import loads_ast, identity, loads, dedent\n", " except:\n", - " from compile import __IPYTHON__, export, Compile, AST\n", - " from capture import capture_output" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ + " from capture import capture_output\n", + " from decoder import loads_ast, identity, loads, dedent\n", + "\n", " import inspect, sys\n", " from importlib.machinery import SourceFileLoader\n", " try: \n", @@ -76,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -109,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -136,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -158,7 +151,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -168,7 +161,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -176,6 +169,11 @@ " \"\"\"A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.\"\"\"\n", " EXTENSION_SUFFIXES = '.ipynb',\n", " \n", + " _compile = staticmethod(compile)\n", + " _loads = staticmethod(loads)\n", + " _transform = staticmethod(dedent)\n", + " _ast_transform = staticmethod(identity)\n", + " \n", " def __init__(\n", " self, fullname=None, path=None, *, \n", " stdout=False, stderr=False, display=False, \n", @@ -208,9 +206,10 @@ " \n", " def __exit__(self, *excepts): remove_one_path_hook(type(self)), super().__exit__(*excepts)\n", " \n", - " def source_to_code(Notebook, data, path):\n", - " with StringIO(data.decode('utf-8')) as stream:\n", - " return Compile().from_file(stream, filename=Notebook.path, name=Notebook.name)\n", + " def source_to_code(self, data, path):\n", + " return self._compile(loads_ast(\n", + " data.decode('utf-8'), loads=self._loads, transform=self._transform, ast_transform=self._ast_transform\n", + " ), path or '', 'exec')\n", " \n", " def from_filename(self, file):\n", " \"\"\"Load a python module or notebook from a file location.\n", @@ -247,8 +246,8 @@ "\n", " module.__output__ = captured\n", " return module\n", - " \n", - " __call__ = from_filename " + " \n", + " __call__ = from_filename" ] }, { @@ -260,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -292,7 +291,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -315,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -334,13 +333,15 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": { "scrolled": false }, "outputs": [], "source": [ " if __name__ == '__main__':\n", + " try: from utils.export import export\n", + " except: from .utils.export import export\n", " export('loader.ipynb', '../loader.py')\n", " __import__('doctest').testmod(Notebook()('loader.ipynb'))" ] diff --git a/src/importnb/notebooks/nbtest.ipynb b/src/importnb/notebooks/nbtest.ipynb index 95a17b1..ad6cff2 100644 --- a/src/importnb/notebooks/nbtest.ipynb +++ b/src/importnb/notebooks/nbtest.ipynb @@ -19,9 +19,9 @@ "outputs": [], "source": [ " try:\n", - " from .loader import Notebook, export\n", + " from .loader import Notebook\n", " except:\n", - " from loader import Notebook, export\n", + " from loader import Notebook\n", "\n", " from unittest import TestProgram, TestCase\n", " from doctest import DocTestSuite\n", diff --git a/src/importnb/notebooks/parameterize.ipynb b/src/importnb/notebooks/parameterize.ipynb index 0d9088f..1564548 100644 --- a/src/importnb/notebooks/parameterize.ipynb +++ b/src/importnb/notebooks/parameterize.ipynb @@ -7,9 +7,11 @@ "outputs": [], "source": [ " try:\n", - " from .loader import Notebook, export\n", + " from .loader import Notebook\n", + " from .decoder import loads_ast\n", " except:\n", - " from loader import Notebook, export\n", + " from loader import Notebook\n", + " from decoder import loads_ast\n", " from inspect import getsource\n", " \n", " from types import ModuleType" @@ -64,26 +66,6 @@ "execution_count": 3, "metadata": {}, "outputs": [], - "source": [ - " def combine_input_strings(nb):\n", - " cells = nb['cells']\n", - " new_cells = []\n", - " for cell in cells:\n", - " if cell['cell_type'] == 'code':\n", - " source = cell['source']\n", - " if isinstance(source, list):\n", - " cell['source'] = ''.join(source)\n", - " \n", - " new_cells.append(cell)\n", - " nb['cells'] = new_cells\n", - " return nb" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], "source": [ " class Parameterize:\n", " \"\"\"Parameterize takes a module, filename, or notebook dictionary and returns callable object that parameterizes the notebook module.\n", @@ -102,32 +84,23 @@ " \n", " if isinstance(object, ModuleType):\n", " self.__file__ = object.__file__\n", - " object = loads(getsource(object))\n", - " \n", - " \n", - " if isinstance(object, str):\n", + " object = getsource(object)\n", + " elif isinstance(object, (Path, str)):\n", " self.__file__ = object\n", - " with open(object) as f: \n", - " self.__notebook__ = load(f)\n", - " elif isinstance(object, dict):\n", - " self.__notebook__ = object\n", + " with open(str(object)) as f: \n", + " object = f.read()\n", + " elif isinstance(object, dict): ...\n", " else: raise ValueError(\"object must be a module, file string, or dict.\")\n", " \n", - " \n", - " self.__notebook__ = combine_input_strings(self.__notebook__)\n", - " \n", - " with capture_output(stdout=False, stderr=False) as output:\n", - " self.__variables__, self.__ast__ = \\\n", - " FreeStatementFinder()(AST().from_notebook_node(self.__notebook__))\n", - " self.__output__ = output\n", + " self.__variables__, self.__ast__ = \\\n", + " FreeStatementFinder()(loads_ast(object))\n", " self.__signature__ = self.vars_to_sig(**self.__variables__)\n", - " # Parameterize.__doc__ = docify(Parameterize.__notebook__)\n", "\n", " def __call__(self, **dict):\n", " self = __import__('copy').copy(self)\n", " self.__dict__.update(self.__variables__)\n", " self.__dict__.update(dict)\n", - " exec(AST(filename=self.__file__).compile(self.__ast__), *[self.__dict__]*2)\n", + " exec(compile(self.__ast__, self.__file__ or '', 'exec'), *[self.__dict__]*2)\n", " return self\n", " \n", " def interact(Parameterize): \n", @@ -138,10 +111,7 @@ " def vars_to_sig(**vars):\n", " \"\"\"Create a signature for a dictionary of names.\"\"\"\n", " from inspect import Parameter, Signature\n", - " return Signature([Parameter(str, Parameter.KEYWORD_ONLY, default = vars[str]) for str in vars])\n", - " \n", - " try: from importnb.loader import AST\n", - " except: from importnb.loader import AST" + " return Signature([Parameter(str, Parameter.KEYWORD_ONLY, default = vars[str]) for str in vars])\n" ] }, { @@ -153,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -172,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -192,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -223,7 +193,7 @@ " assert new.param is 10 and new.extraparam is 42\n", "Expecting nothing\n", "ok\n", - "12 items had no tests:\n", + "11 items had no tests:\n", " __main__\n", " __main__.FreeStatementFinder\n", " __main__.FreeStatementFinder.__call__\n", @@ -235,12 +205,11 @@ " __main__.Parameterize.__init__\n", " __main__.Parameterize.interact\n", " __main__.Parameterize.vars_to_sig\n", - " __main__.combine_input_strings\n", "3 items passed all tests:\n", " 3 tests in __main__.__test__.default\n", " 1 tests in __main__.__test__.imports\n", " 2 tests in __main__.__test__.reuse\n", - "6 tests in 15 items.\n", + "6 tests in 14 items.\n", "6 passed and 0 failed.\n", "Test passed.\n" ] @@ -267,14 +236,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": { "scrolled": false }, "outputs": [], "source": [ " if __name__ == '__main__':\n", - " export('parameterize.ipynb', '../parameterize.py')\n", + " # export('parameterize.ipynb', '../parameterize.py')\n", " __import__('doctest').testmod()" ] } diff --git a/src/importnb/notebooks/utils/export.ipynb b/src/importnb/notebooks/utils/export.ipynb index d3d1e56..c92f524 100644 --- a/src/importnb/notebooks/utils/export.ipynb +++ b/src/importnb/notebooks/utils/export.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# The `compile` module\n", + "# The `export` module\n", "\n", "...provides compatibility for Python and IPython through [`compile_python`](compile_python.ipynb) and [`compile_ipython`](compile_ipython.ipynb), respectively. \n", "\n", @@ -54,15 +54,20 @@ }, "outputs": [ { - "ename": "TypeError", - "evalue": "testmod: module required; ", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexport\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mthis\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexport\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mthis\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0m__import__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'doctest'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtestmod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/anaconda/envs/p6/lib/python3.6/doctest.py\u001b[0m in \u001b[0;36mtestmod\u001b[0;34m(m, name, globs, verbose, report, optionflags, extraglobs, raise_on_error, exclude_empty)\u001b[0m\n\u001b[1;32m 1934\u001b[0m \u001b[0;31m# Check that we were actually given a module.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1935\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mismodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1936\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"testmod: module required; %r\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1937\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1938\u001b[0m \u001b[0;31m# If no name was given, then use the module's name.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: testmod: module required; " + "name": "stdout", + "output_type": "stream", + "text": [ + "Trying:\n", + " from importnb.utils.export import export\n", + "Expecting nothing\n", + "ok\n", + "1 items had no tests:\n", + " export.export\n", + "1 items passed all tests:\n", + " 1 tests in export\n", + "1 tests in 2 items.\n", + "1 passed and 0 failed.\n", + "Test passed.\n" ] } ], diff --git a/src/importnb/parameterize.py b/src/importnb/parameterize.py index 3efd861..adf1097 100644 --- a/src/importnb/parameterize.py +++ b/src/importnb/parameterize.py @@ -2,9 +2,11 @@ # coding: utf-8 try: - from .loader import Notebook, export + from .loader import Notebook + from .decoder import loads_ast except: - from loader import Notebook, export + from loader import Notebook + from decoder import loads_ast from inspect import getsource from types import ModuleType @@ -57,20 +59,6 @@ def __call__(FreeStatement, nodes): """ -def combine_input_strings(nb): - cells = nb["cells"] - new_cells = [] - for cell in cells: - if cell["cell_type"] == "code": - source = cell["source"] - if isinstance(source, list): - cell["source"] = "".join(source) - - new_cells.append(cell) - nb["cells"] = new_cells - return nb - - class Parameterize: """Parameterize takes a module, filename, or notebook dictionary and returns callable object that parameterizes the notebook module. @@ -88,32 +76,26 @@ def __init__(self, object=None): if isinstance(object, ModuleType): self.__file__ = object.__file__ - object = loads(getsource(object)) - - if isinstance(object, str): + object = getsource(object) + elif isinstance(object, (Path, str)): self.__file__ = object - with open(object) as f: - self.__notebook__ = load(f) + with open(str(object)) as f: + object = f.read() elif isinstance(object, dict): - self.__notebook__ = object + ... else: raise ValueError("object must be a module, file string, or dict.") - self.__notebook__ = combine_input_strings(self.__notebook__) - - with capture_output(stdout=False, stderr=False) as output: - self.__variables__, self.__ast__ = FreeStatementFinder()( - AST().from_notebook_node(self.__notebook__) - ) - self.__output__ = output + self.__variables__, self.__ast__ = FreeStatementFinder()(loads_ast(object)) self.__signature__ = self.vars_to_sig(**self.__variables__) - # Parameterize.__doc__ = docify(Parameterize.__notebook__) def __call__(self, **dict): self = __import__("copy").copy(self) self.__dict__.update(self.__variables__) self.__dict__.update(dict) - exec(AST(filename=self.__file__).compile(self.__ast__), *[self.__dict__] * 2) + exec( + compile(self.__ast__, self.__file__ or "", "exec"), *[self.__dict__] * 2 + ) return self def interact(Parameterize): @@ -130,12 +112,6 @@ def vars_to_sig(**vars): ) -try: - from importnb.loader import AST -except: - from importnb.loader import AST - - """#### Examples that do work """ @@ -180,5 +156,5 @@ def vars_to_sig(**vars): if __name__ == "__main__": - export("parameterize.ipynb", "../parameterize.py") + # export('parameterize.ipynb', '../parameterize.py') __import__("doctest").testmod() diff --git a/src/importnb/tests/test_importnb.ipynb b/src/importnb/tests/test_importnb.ipynb index 5d8af34..62501d3 100644 --- a/src/importnb/tests/test_importnb.ipynb +++ b/src/importnb/tests/test_importnb.ipynb @@ -18,8 +18,8 @@ "from pathlib import Path\n", "import shutil, os, functools, sys\n", "from pytest import fixture, mark\n", - "from importnb.loader import __IPYTHON__\n", "import warnings\n", + "__IPYTHON__ = bool(globals().get('__IPYTHON__', False))\n", "print(__IPYTHON__)" ] }, diff --git a/src/importnb/utils/export.py b/src/importnb/utils/export.py index de57790..6381f09 100644 --- a/src/importnb/utils/export.py +++ b/src/importnb/utils/export.py @@ -1,7 +1,7 @@ # coding: utf-8 -"""# The `compile` module +"""# The `export` module ...provides compatibility for Python and IPython through [`compile_python`](compile_python.ipynb) and [`compile_ipython`](compile_ipython.ipynb), respectively. diff --git a/src/notebooks/compile.ipynb b/src/notebooks/compile.ipynb deleted file mode 100644 index 373acbb..0000000 --- a/src/notebooks/compile.ipynb +++ /dev/null @@ -1,244 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The `compile` module\n", - "\n", - "...provides compatibility for Python and IPython through [`compile_python`](compile_python.ipynb) and [`compile_ipython`](compile_ipython.ipynb), respectively. \n", - "\n", - " >>> from importnb.compile import __IPYTHON__, Compiler, PythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " import ast, sys\n", - " from pathlib import Path\n", - " \n", - " __file__ = globals().get('__file__', 'compile.ipynb')\n", - " __nb__ = __file__.replace('src/importnb', 'src/notebooks')\n", - "\n", - " __IPYTHON__ = False\n", - "\n", - " try:\n", - " from IPython import get_ipython\n", - " if not get_ipython(): ...\n", - " else: __IPYTHON__ = True\n", - " except: ...\n", - "\n", - " if __IPYTHON__:\n", - " try:\n", - " from .compile_ipython import Compiler, PythonExporter\n", - " except:\n", - " from compile_ipython import Compiler, PythonExporter\n", - " else:\n", - " try:\n", - " from .compile_python import Compiler, PythonExporter\n", - " except:\n", - " from compile_python import Compiler, PythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " class ImportNbStyleExporter(PythonExporter):\n", - " PythonExporter.exclude_input_prompt = True\n", - " def from_notebook_node(self, nb, resources=None, **kw):\n", - " code, resources = super().from_notebook_node(nb, resources=resources, **kw)\n", - " try:\n", - " from black import format_str\n", - " except:\n", - " format_str = lambda x, i: x\n", - " return format_str(code, 100), resources" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - " def export(file, to=None): \n", - " from pathlib import Path\n", - " exporter = ImportNbStyleExporter()\n", - " code = exporter.from_filename(file)[0]\n", - " if to: Path(to).with_suffix(exporter.file_extension).write_text(code)\n", - " return code " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - " class Code(PythonExporter, Compiler):\n", - " \"\"\"An exporter than returns transforms a NotebookNode through the InputSplitter.\n", - " \n", - " >>> assert type(Code().from_filename(Path(__nb__).with_suffix('.ipynb'))) is __import__('nbformat').NotebookNode\"\"\"\n", - " def __init__(\n", - " self, \n", - " filename='', name='__main__'\n", - " ): \n", - " PythonExporter.__init__(self) \n", - " Compiler.__init__(self)\n", - " self.filename = filename\n", - " self.name = name\n", - " \n", - " def from_file(Code, file_stream, resources=None, **dict): \n", - " for str in ('name', 'filename'): setattr(Code, str, dict.pop(str, getattr(Code, str)))\n", - " return super().from_file(file_stream, resources, **dict)\n", - " \n", - " def from_filename(Code, filename, resources=None, **dict):\n", - " Code.filename, Code.name = filename, Path(filename).stem\n", - " return super().from_filename(filename, resources, **dict)\n", - "\n", - " def from_notebook_node(Code, nb, resources=None, **dict): \n", - " for index, cell in enumerate(nb['cells']):\n", - " if cell['cell_type'] == 'code':\n", - " if isinstance(cell['source'], list):\n", - " cell['source'] = ''.join(cell['source'])\n", - " cell['source'] = Code.from_code_cell(cell, **dict)\n", - " return nb\n", - " \n", - " def from_code_cell(Code, cell, **dict): \n", - " return Code.transform(cell['source'])" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - " class AST(Code):\n", - " \"\"\"An exporter than returns parsed ast.\n", - " \n", - " >>> assert type(AST().from_filename(Path(__nb__).with_suffix('.ipynb'))) is ast.Module\"\"\"\n", - " def from_notebook_node(AST, nb, resource=None, **dict): \n", - " module = AST.ast_transform(ast.fix_missing_locations(ast.Module(body=sum((\n", - " AST.ast_parse(\n", - " AST.from_code_cell(cell, **dict), \n", - " lineno=cell['metadata'].get('lineno', 1)\n", - " ).body \n", - " for cell in super().from_notebook_node(nb, resource, **dict)['cells'] \n", - " if cell['cell_type']=='code'\n", - " ), []))))\n", - "\n", - " # The module ast node has a docstring parameter.\n", - " # https://docs.python.org/3.7/whatsnew/3.7.html#changes-in-the-python-api\n", - " \n", - " return module" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - " class Compile(AST):\n", - " \"\"\"An exporter that returns compiled and cached bytecode.\n", - " \n", - " >>> assert Compile().from_filename(Path(__nb__).with_suffix('.ipynb'))\"\"\" \n", - " def from_notebook_node(Compile, nb, resources: dict=None, **dict):\n", - " return Compile.compile(super().from_notebook_node(nb, resources, **dict))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trying:\n", - " from importnb.compile import __IPYTHON__, Compiler, PythonExporter\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert type(AST().from_filename(Path(__nb__).with_suffix('.ipynb'))) is ast.Module\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert type(Code().from_filename(Path(__nb__).with_suffix('.ipynb'))) is __import__('nbformat').NotebookNode\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert Compile().from_filename(Path(__nb__).with_suffix('.ipynb'))\n", - "Expecting nothing\n", - "ok\n", - "10 items had no tests:\n", - " compile.AST.from_notebook_node\n", - " compile.Code.__init__\n", - " compile.Code.from_code_cell\n", - " compile.Code.from_file\n", - " compile.Code.from_filename\n", - " compile.Code.from_notebook_node\n", - " compile.Compile.from_notebook_node\n", - " compile.ImportNbStyleExporter\n", - " compile.ImportNbStyleExporter.from_notebook_node\n", - " compile.export\n", - "4 items passed all tests:\n", - " 1 tests in compile\n", - " 1 tests in compile.AST\n", - " 1 tests in compile.Code\n", - " 1 tests in compile.Compile\n", - "4 tests in 14 items.\n", - "4 passed and 0 failed.\n", - "Test passed.\n" - ] - } - ], - "source": [ - " if __name__ == '__main__':\n", - " export('compile.ipynb', '../importnb/compile.py')\n", - " try:\n", - " from . import compile\n", - " except:\n", - " import compile\n", - " __import__('doctest').testmod(compile, verbose=2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/notebooks/compile_ipython.ipynb b/src/notebooks/compile_ipython.ipynb deleted file mode 100644 index e1870d0..0000000 --- a/src/notebooks/compile_ipython.ipynb +++ /dev/null @@ -1,166 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The IPython compiler" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - " try:\n", - " from .decoder import load\n", - " except:\n", - " from decoder import load\n", - "\n", - " from nbconvert.exporters.python import PythonExporter as _PythonExporter\n", - " from nbconvert.exporters.notebook import NotebookExporter\n", - " from nbformat import from_dict\n", - " from IPython.core.compilerop import CachingCompiler\n", - " from IPython.core.inputsplitter import IPythonInputSplitter\n", - " import ast\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - " class IpythonCompiler(CachingCompiler):\n", - " \"\"\"{Shell} provides the IPython machinery to objects.\"\"\"\n", - " transform = staticmethod(IPythonInputSplitter().transform_cell)\n", - " @property\n", - " def ip(Compiler): \n", - " \"\"\"The current interactive shell\"\"\"\n", - " from IPython import get_ipython\n", - " from IPython.core.interactiveshell import InteractiveShell\n", - " return get_ipython() or InteractiveShell()\n", - " \n", - " def ast_transform(Compiler, node):\n", - " for visitor in Compiler.ip.ast_transformers: \n", - " node = visitor.visit(node)\n", - " return node\n", - " \n", - " def compile(Compiler, ast): \n", - " \"\"\"Compile AST to bytecode using the an IPython compiler.\"\"\"\n", - " return (Compiler.ip and Compiler.ip.compile or CachingCompiler())(ast, Compiler.filename, 'exec')\n", - " \n", - " def ast_parse(Compiler, source, filename='', symbol='exec', lineno=0): \n", - " return ast.increment_lineno(super().ast_parse(source, Compiler.filename, 'exec'), lineno)\n", - " \n", - " Compiler = IpythonCompiler" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - " class IpythonPythonExporter(_PythonExporter):\n", - " def from_file(self, file_stream, resources=None, **kw):\n", - " return self.from_notebook_node(from_dict(load(file_stream)), resources, **kw)\n", - " PythonExporter = IpythonPythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - " if __name__ == '__main__':\n", - " try: from .compile import export\n", - " except: from compile import export\n", - " export('compile_ipython.ipynb', '../importnb/compile_ipython.py')\n", - " __import__('doctest').testmod()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - }, - "toc": { - "colors": { - "hover_highlight": "#DAA520", - "running_highlight": "#FF0000", - "selected_highlight": "#FFD700" - }, - "moveMenuLeft": true, - "nav_menu": { - "height": "29px", - "width": "252px" - }, - "navigate_menu": true, - "number_sections": true, - "sideBar": true, - "threshold": 4, - "toc_cell": false, - "toc_section_display": "block", - "toc_window_display": false, - "widenNotebook": false - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/notebooks/compile_python.ipynb b/src/notebooks/compile_python.ipynb deleted file mode 100644 index a3343d1..0000000 --- a/src/notebooks/compile_python.ipynb +++ /dev/null @@ -1,177 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The Python compile module" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " import ast, sys\n", - " try:\n", - " from .decoder import load\n", - " except:\n", - " from decoder import load\n", - " from pathlib import Path\n", - " from textwrap import dedent\n", - " from codeop import Compile" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " class PythonCompiler(Compile):\n", - " \"\"\"{Shell} provides the IPython machinery to objects.\"\"\"\n", - " def ast_transform(Compiler, node): return node\n", - " \n", - " @property\n", - " def transform(Compiler): return dedent\n", - "\n", - " def compile(Compiler, ast): return compile(ast, Compiler.filename, 'exec')\n", - " \n", - " def ast_parse(Compiler, source, filename='', symbol='exec', lineno=0): \n", - " return ast.increment_lineno(ast.parse(source, Compiler.filename, 'exec'), lineno)\n", - " \n", - " Compiler = PythonCompiler" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - " class PythonNotebookExporter:\n", - " def from_file(self, file_stream, resources=None, **kw):\n", - " return self.from_notebook_node(load(file_stream), resources, **kw) \n", - " \n", - " def from_filename(self, filename, resources=None, **dict):\n", - " with open(filename, 'r') as file_stream:\n", - " return self.from_file(file_stream, resources, **dict)\n", - " \n", - " def from_notebook_node(self, nb, resources=None, **dict): \n", - " return nb, resources\n", - " NotebookExporter = PythonNotebookExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - " class PythonPythonExporter(NotebookExporter):\n", - " def from_notebook_node(self, nb, resources=None, **dict):\n", - " nb, resources = super().from_notebook_node(nb, resources, **dict)\n", - " for i, cell in enumerate(nb['cells']):\n", - " if isinstance(cell['source'], list):\n", - " nb['cells'][i]['source'] = ''.join(cell['source'])\n", - " return (\"\\n\"*2).join(\n", - " dedent(cell['source']) for cell in nb['cells'] if cell['cell_type'] == 'code'\n", - " ), resources\n", - " PythonExporter = PythonPythonExporter" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - " if __name__ == '__main__':\n", - " try: from .compile import export\n", - " except: from compile import export\n", - " export('compile_python.ipynb', '../importnb/compile_python.py')\n", - " __import__('doctest').testmod()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - }, - "toc": { - "colors": { - "hover_highlight": "#DAA520", - "running_highlight": "#FF0000", - "selected_highlight": "#FFD700" - }, - "moveMenuLeft": true, - "nav_menu": { - "height": "29px", - "width": "252px" - }, - "navigate_menu": true, - "number_sections": true, - "sideBar": true, - "threshold": 4, - "toc_cell": false, - "toc_section_display": "block", - "toc_window_display": false, - "widenNotebook": false - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/notebooks/nbtest.ipynb b/src/notebooks/nbtest.ipynb deleted file mode 100644 index d4347ca..0000000 --- a/src/notebooks/nbtest.ipynb +++ /dev/null @@ -1,141 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Load and test notebooks\n", - "\n", - "New ideas may include tests in a notebook. The `importnb.test.Test` context will `doctest` and `unittest` a notebook.\n", - "\n", - " >>> from importnb import NotebookTest\n", - " >>> assert NotebookTest" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " try:\n", - " from .loader import Notebook, export\n", - " except:\n", - " from loader import Notebook, export\n", - "\n", - " from unittest import TestProgram, TestCase\n", - " from doctest import DocTestSuite\n", - " \n", - " __file__ = globals().get('__file__', 'nbtest.ipynb')" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " def attach_doctest(module):\n", - " \"\"\"A function to include doctests in a unittest suite.\n", - " \"\"\"\n", - " def load_tests(loader, tests, ignore):\n", - " tests.addTests(DocTestSuite(module))\n", - " return tests\n", - " module.load_tests = load_tests\n", - " return module" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - " def testmod(module, extras=\"\", doctest=True, exit=True, verbosity=1, failfast=None, catchbreak=None):\n", - " \"\"\"Test a module using unittest, include the docstrings by default.\"\"\"\n", - " if doctest:\n", - " attach_doctest(module)\n", - " try:\n", - " TestProgram(module, argv=' '.join(('discover', extras)).split(), exit=exit, verbosity=verbosity, failfast=failfast, catchbreak=catchbreak)\n", - " except SystemExit: ...\n", - " return module" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - " class NotebookTest(Notebook):\n", - " def exec_module(self, module):\n", - " super().exec_module(module)\n", - " testmod(module)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - " class _test(TestCase):\n", - " def test_importnb_test(self):\n", - " assert True" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "..\n", - "----------------------------------------------------------------------\n", - "Ran 2 tests in 0.003s\n", - "\n", - "OK\n", - "..\n", - "----------------------------------------------------------------------\n", - "Ran 2 tests in 0.003s\n", - "\n", - "OK\n" - ] - } - ], - "source": [ - " if __name__ == '__main__':\n", - " export('nbtest.ipynb', '../importnb/nbtest.py')\n", - " __import__('doctest').testmod(Notebook.from_filename('nbtest.ipynb'))\n", - " m = NotebookTest.from_filename(__file__)\n", - " testmod(m, \"-f\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/notebooks/utils/nbdoctest.ipynb b/src/notebooks/utils/nbdoctest.ipynb deleted file mode 100644 index b4f0f7e..0000000 --- a/src/notebooks/utils/nbdoctest.ipynb +++ /dev/null @@ -1,92 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython import get_ipython\n", - "from doctest import OPTIONFLAGS_BY_NAME, testfile, testmod, FAIL_FAST\n", - "import os, argparse\n", - "from importnb import Notebook" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def _test():\n", - " parser = argparse.ArgumentParser(description=\"doctest runner\")\n", - " parser.add_argument('-v', '--verbose', action='store_true', default=False,\n", - " help='print very verbose output for all tests')\n", - " parser.add_argument('-o', '--option', action='append',\n", - " choices=OPTIONFLAGS_BY_NAME.keys(), default=[],\n", - " help=('specify a doctest option flag to apply'\n", - " ' to the test run; may be specified more'\n", - " ' than once to apply multiple options'))\n", - " parser.add_argument('-f', '--fail-fast', action='store_true',\n", - " help=('stop running tests after first failure (this'\n", - " ' is a shorthand for -o FAIL_FAST, and is'\n", - " ' in addition to any other -o options)'))\n", - " parser.add_argument('file', nargs='+',\n", - " help='file containing the tests to run')\n", - " args = parser.parse_args()\n", - " testfiles = args.file\n", - " # Verbose used to be handled by the \"inspect argv\" magic in DocTestRunner,\n", - " # but since we are using argparse we are passing it manually now.\n", - " verbose = args.verbose\n", - " options = 0\n", - " for option in args.option:\n", - " options |= OPTIONFLAGS_BY_NAME[option]\n", - " if args.fail_fast:\n", - " options |= FAIL_FAST\n", - " for filename in testfiles:\n", - " if any(map(filename.endswith, (\".py\", \".ipynb\"))):\n", - " # It is a module -- insert its dir into sys.path and try to\n", - " # import it. If it is part of a package, that possibly\n", - " # won't work because of package imports.\n", - " failures, _ = testmod(Notebook.from_filename(filename), verbose=verbose, optionflags=options)\n", - " else:\n", - " failures, _ = testfile(filename, module_relative=False,\n", - " verbose=verbose, optionflags=options)\n", - " if failures:\n", - " return 1\n", - " return 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if __name__ == '__main__':\n", - " _test()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}