From 3ad30204857a1cf50c4f4a8a811611e35a70d084 Mon Sep 17 00:00:00 2001 From: Simon Gomizelj Date: Fri, 23 Feb 2018 17:39:44 -0500 Subject: [PATCH 1/4] Ignore py3*_only tests everywhere --- conftest.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/conftest.py b/conftest.py index 54ba2e6dc..a3e531f6e 100644 --- a/conftest.py +++ b/conftest.py @@ -1,15 +1,20 @@ import pytest -import hy +import hy # NOQA import os from hy._compat import PY3, PY35, PY36 + NATIVE_TESTS = os.path.join("", "tests", "native_tests", "") + +def pytest_ignore_collect(path, config): + return (("py3_only" in path.basename and not PY3) + or ("py35_only" in path.basename and not PY35) + or ("py36_only" in path.basename and not PY36)) + + def pytest_collect_file(parent, path): if (path.ext == ".hy" - and NATIVE_TESTS in path.dirname + os.sep - and path.basename != "__init__.hy" - and not ("py3_only" in path.basename and not PY3) - and not ("py35_only" in path.basename and not PY35) - and not ("py36_only" in path.basename and not PY36)): + and NATIVE_TESTS in path.dirname + os.sep + and path.basename != "__init__.hy"): return pytest.Module(path, parent) From 251a053cd227b0fc7ca183e080c6263f12b5b92c Mon Sep 17 00:00:00 2001 From: Simon Gomizelj Date: Fri, 16 Feb 2018 15:07:57 -0500 Subject: [PATCH 2/4] Replace the loader with a more compatable implementation Add the necessary bits to provide a Python 3.4+ compliant import machinery. This means we now have our own PathFinder, which we register as in sys.meta_path, our own FileFinder, and our own Loader. Add a compatability later that builds on the Python 3 machinery to make it best-effort backwards compatible with Python 2. --- hy/__init__.py | 4 +- hy/_compat.py | 18 +- hy/cmdline.py | 90 +++------ hy/compiler.py | 10 +- hy/core/language.hy | 2 +- hy/errors.py | 8 - hy/importer.py | 294 ---------------------------- hy/importlib/__init__.py | 116 +++++++++++ hy/importlib/bytecode.py | 117 +++++++++++ hy/importlib/compat.py | 110 +++++++++++ hy/importlib/loader.py | 77 ++++++++ hy/importlib/machinery.py | 89 +++++++++ hy/importlib/util.py | 56 ++++++ tests/compilers/test_ast.py | 9 +- tests/native_tests/native_macros.hy | 28 +-- tests/test_bin.py | 16 +- 16 files changed, 638 insertions(+), 406 deletions(-) delete mode 100644 hy/importer.py create mode 100644 hy/importlib/__init__.py create mode 100644 hy/importlib/bytecode.py create mode 100644 hy/importlib/compat.py create mode 100644 hy/importlib/loader.py create mode 100644 hy/importlib/machinery.py create mode 100644 hy/importlib/util.py diff --git a/hy/__init__.py b/hy/__init__.py index 42d3133de..edba20239 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -8,9 +8,9 @@ from hy.models import HyExpression, HyInteger, HyKeyword, HyComplex, HyString, HyBytes, HySymbol, HyFloat, HyDict, HyList, HySet, HyCons # NOQA -import hy.importer # NOQA +import hy.importlib # NOQA # we import for side-effects. from hy.core.language import read, read_str, mangle, unmangle # NOQA -from hy.importer import hy_eval as eval # NOQA +from hy.importlib import hy_eval as eval # NOQA diff --git a/hy/_compat.py b/hy/_compat.py index c40e44df7..a9d8d2bdd 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -6,18 +6,7 @@ import __builtin__ as builtins except ImportError: import builtins # NOQA -try: - from py_compile import MAGIC, wr_long -except ImportError: - # py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4 - from importlib.util import MAGIC_NUMBER as MAGIC # NOQA - - def wr_long(f, x): - """Internal; write a 32-bit int to a file in little-endian order.""" - f.write(bytes([x & 0xff, - (x >> 8) & 0xff, - (x >> 16) & 0xff, - (x >> 24) & 0xff])) + import sys, keyword PY3 = sys.version_info[0] >= 3 @@ -60,3 +49,8 @@ def isidentifier(x): except T.TokenError: return False return len(tokens) == 2 and tokens[0][0] == T.NAME + +try: + FileNotFoundError = FileNotFoundError +except NameError: + FileNotFoundError = IOError diff --git a/hy/cmdline.py b/hy/cmdline.py index 151b6eba0..14ec44565 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -10,26 +10,19 @@ import sys import os import importlib +import runpy import astor.code_gen import hy - +from hy.compiler import HyTypeError +from hy.completer import completion, Completer +from hy.importlib import hy_eval, hy_parse, hy_compile from hy.lex import LexException, PrematureEndOfInput from hy.lex.parser import mangle -from hy.compiler import HyTypeError -from hy.importer import (hy_eval, import_buffer_to_module, - import_file_to_ast, import_file_to_hst, - import_buffer_to_ast, import_buffer_to_hst) -from hy.completer import completion -from hy.completer import Completer - -from hy.errors import HyIOError - from hy.macros import macro, require from hy.models import HyExpression, HyString, HySymbol - -from hy._compat import builtins, PY3 +from hy._compat import builtins, PY3, FileNotFoundError class HyQuitter(object): @@ -39,8 +32,6 @@ def __init__(self, name): def __repr__(self): return "Use (%s) or Ctrl-D (i.e. EOF) to exit" % (self.name) - __str__ = __repr__ - def __call__(self, code=None): try: sys.stdin.close() @@ -48,6 +39,7 @@ def __call__(self, code=None): pass raise SystemExit(code) + builtins.quit = HyQuitter('quit') builtins.exit = HyQuitter('exit') @@ -77,7 +69,7 @@ def runsource(self, source, filename='', symbol='single'): global SIMPLE_TRACEBACKS try: try: - do = import_buffer_to_hst(source) + do = hy_parse(source) except PrematureEndOfInput: return True except LexException as e: @@ -95,7 +87,7 @@ def ast_callback(main_ast, expr_ast): new_ast = ast.Module(main_ast.body + [ast.Expr(expr_ast.body)]) print(astor.to_source(new_ast)) - value = hy_eval(do, self.locals, "__console__", + value = hy_eval(do, self.locals, "__main__", ast_callback) except HyTypeError as e: if e.source is None: @@ -175,7 +167,6 @@ def ideas_macro(ETname): """)]) -require("hy.cmdline", "__console__", all_macros=True) require("hy.cmdline", "__main__", all_macros=True) SIMPLE_TRACEBACKS = True @@ -192,26 +183,7 @@ def pretty_error(func, *args, **kw): def run_command(source): - pretty_error(import_buffer_to_module, "__main__", source) - return 0 - - -def run_module(mod_name): - from hy.importer import MetaImporter - pth = MetaImporter().find_on_path(mod_name) - if pth is not None: - sys.argv = [pth] + sys.argv - return run_file(pth) - - print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name), - file=sys.stderr) - return 1 - - -def run_file(filename): - from hy.importer import import_file_to_module - pretty_error(import_file_to_module, "__main__", filename) - return 0 + hy_eval(hy_parse(source)) def run_repl(hr=None, **kwargs): @@ -219,7 +191,7 @@ def run_repl(hr=None, **kwargs): sys.ps1 = "=> " sys.ps2 = "... " - namespace = {'__name__': '__console__', '__doc__': ''} + namespace = {'__name__': '__main__', '__doc__': ''} with completion(Completer(namespace)): @@ -316,7 +288,8 @@ def cmdline_handler(scriptname, argv): if options.mod: # User did "hy -m ..." - return run_module(options.mod) + runpy.run_module(options.mod, run_name='__main__', alter_sys=True) + return 0 if options.icommand: # User did "hy -i ..." @@ -331,10 +304,11 @@ def cmdline_handler(scriptname, argv): else: # User did "hy " try: - return run_file(options.args[0]) - except HyIOError as e: - print("hy: Can't open file '{0}': [Errno {1}] {2}\n".format( - e.filename, e.errno, e.strerror), file=sys.stderr) + runpy.run_path(options.args[0], run_name='__main__') + return 0 + except FileNotFoundError as e: + print("hy: Can't open file '{0}': [Errno {1}] {2}".format( + e.filename, e.errno, e.strerror), file=sys.stderr) sys.exit(e.errno) # User did NOTHING! @@ -343,12 +317,12 @@ def cmdline_handler(scriptname, argv): # entry point for cmd line script "hy" def hy_main(): + sys.path.insert(0, "") sys.exit(cmdline_handler("hy", sys.argv)) # entry point for cmd line script "hyc" def hyc_main(): - from hy.importer import write_hy_as_pyc parser = argparse.ArgumentParser(prog="hyc") parser.add_argument("files", metavar="FILE", nargs='+', help="file to compile") @@ -358,12 +332,14 @@ def hyc_main(): for file in options.files: try: - print("Compiling %s" % file) - pretty_error(write_hy_as_pyc, file) - except IOError as x: - print("hyc: Can't open file '{0}': [Errno {1}] {2}\n".format( - x.filename, x.errno, x.strerror), file=sys.stderr) - sys.exit(x.errno) + print("Compiling {}".format(file)) + with open(file): + # TODO + pass + except FileNotFoundError as e: + print("hyc: Can't open file '{0}': [Errno {1}] {2}".format( + e.filename, e.errno, e.strerror), file=sys.stderr) + sys.exit(e.errno) # entry point for cmd line script "hy2py" @@ -387,14 +363,14 @@ def hy2py_main(): options = parser.parse_args(sys.argv[1:]) - stdin_text = None if options.FILE is None or options.FILE == '-': - stdin_text = sys.stdin.read() + source = sys.stdin.read() + else: + with open(options.FILE) as source_file: + source = source_file.read() + hst = (pretty_error(hy_parse, source)) if options.with_source: - hst = (pretty_error(import_file_to_hst, options.FILE) - if stdin_text is None - else pretty_error(import_buffer_to_hst, stdin_text)) # need special printing on Windows in case the # codepage doesn't support utf-8 characters if PY3 and platform.system() == "Windows": @@ -408,9 +384,7 @@ def hy2py_main(): print() print() - _ast = (pretty_error(import_file_to_ast, options.FILE, module_name) - if stdin_text is None - else pretty_error(import_buffer_to_ast, stdin_text, module_name)) + _ast = (pretty_error(hy_compile, hst, module_name)) if options.with_ast: if PY3 and platform.system() == "Windows": _print_for_windows(astor.dump_tree(_ast)) diff --git a/hy/compiler.py b/hy/compiler.py index 9a130f0fe..ad209fb61 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -3,6 +3,8 @@ # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. +from __future__ import absolute_import + from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex, HyString, HyBytes, HySymbol, HyFloat, HyList, HySet, HyDict, HyCons, wrap_value) @@ -15,7 +17,7 @@ str_type, string_types, bytes_type, long_type, PY3, PY35, PY37, raise_empty) from hy.macros import require, macroexpand, tag_macroexpand -import hy.importer +import hy.importlib import hy.inspect import traceback @@ -2120,9 +2122,9 @@ def compile_dispatch_tag_macro(self, expression): @builds("eval-and-compile", "eval-when-compile") def compile_eval_and_compile(self, expression, building): expression[0] = HySymbol("do") - hy.importer.hy_eval(expression, - compile_time_ns(self.module_name), - self.module_name) + hy.importlib.hy_eval(expression, + compile_time_ns(self.module_name), + self.module_name) return (self._compile_branch(expression[1:]) if building == "eval_and_compile" else Result()) diff --git a/hy/core/language.hy b/hy/core/language.hy index f9733424a..09a6eb0d8 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -22,7 +22,7 @@ (import [hy.lex [LexException PrematureEndOfInput tokenize]]) (import [hy.lex.parser [mangle unmangle]]) (import [hy.compiler [HyASTCompiler spoof-positions]]) -(import [hy.importer [hy-eval :as eval]]) +(import [hy.importlib [hy-eval :as eval]]) (defn butlast [coll] "Return an iterator of all but the last item in `coll`." diff --git a/hy/errors.py b/hy/errors.py index a60b09cb5..3425650cb 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -94,11 +94,3 @@ def __str__(self): class HyMacroExpansionError(HyTypeError): pass - - -class HyIOError(HyError, IOError): - """ - Trivial subclass of IOError and HyError, to distinguish between - IOErrors raised by Hy itself as opposed to Hy programs. - """ - pass diff --git a/hy/importer.py b/hy/importer.py deleted file mode 100644 index 1fd8c6d02..000000000 --- a/hy/importer.py +++ /dev/null @@ -1,294 +0,0 @@ -# Copyright 2018 the authors. -# This file is part of Hy, which is free software licensed under the Expat -# license. See the LICENSE. - -from __future__ import absolute_import - -from hy.compiler import hy_compile, HyTypeError -from hy.models import HyObject, HyExpression, HySymbol, replace_hy_obj -from hy.lex import tokenize, LexException -from hy.errors import HyIOError - -from io import open -import re -import marshal -import struct -import imp -import sys -import ast -import inspect -import os -import __future__ - -from hy._compat import PY3, PY37, MAGIC, builtins, long_type, wr_long -from hy._compat import string_types - - -def ast_compile(ast, filename, mode): - """Compile AST. - Like Python's compile, but with some special flags.""" - flags = (__future__.CO_FUTURE_DIVISION | - __future__.CO_FUTURE_PRINT_FUNCTION) - return compile(ast, filename, mode, flags) - - -def import_buffer_to_hst(buf): - """Import content from buf and return a Hy AST.""" - return HyExpression([HySymbol("do")] + tokenize(buf + "\n")) - - -def import_file_to_hst(fpath): - """Import content from fpath and return a Hy AST.""" - try: - with open(fpath, 'r', encoding='utf-8') as f: - buf = f.read() - # Strip the shebang line, if there is one. - buf = re.sub(r'\A#!.*', '', buf) - return import_buffer_to_hst(buf) - except IOError as e: - raise HyIOError(e.errno, e.strerror, e.filename) - - -def import_buffer_to_ast(buf, module_name): - """ Import content from buf and return a Python AST.""" - return hy_compile(import_buffer_to_hst(buf), module_name) - - -def import_file_to_ast(fpath, module_name): - """Import content from fpath and return a Python AST.""" - return hy_compile(import_file_to_hst(fpath), module_name) - - -def import_file_to_module(module_name, fpath, loader=None): - """Import Hy source from fpath and put it into a Python module. - - If there's an up-to-date byte-compiled version of this module, load that - instead. Otherwise, byte-compile the module once we're done loading it, if - we can. - - Return the module.""" - - module = None - - bytecode_path = get_bytecode_path(fpath) - try: - source_mtime = int(os.stat(fpath).st_mtime) - with open(bytecode_path, 'rb') as bc_f: - # The first 4 bytes are the magic number for the version of Python - # that compiled this bytecode. - bytecode_magic = bc_f.read(4) - # Python 3.7 introduced a new flags entry in the header structure. - if PY37: - bc_f.read(4) - # The next 4 bytes, interpreted as a little-endian 32-bit integer, - # are the mtime of the corresponding source file. - bytecode_mtime, = struct.unpack('= source_mtime: - # It's a cache hit. Load the byte-compiled version. - if PY3: - # As of Python 3.6, imp.load_compiled still exists, but it's - # deprecated. So let's use SourcelessFileLoader instead. - from importlib.machinery import SourcelessFileLoader - module = (SourcelessFileLoader(module_name, bytecode_path). - load_module(module_name)) - else: - module = imp.load_compiled(module_name, bytecode_path) - - if not module: - # It's a cache miss, so load from source. - sys.modules[module_name] = None - try: - _ast = import_file_to_ast(fpath, module_name) - module = imp.new_module(module_name) - module.__file__ = os.path.normpath(fpath) - code = ast_compile(_ast, fpath, "exec") - if not os.environ.get('PYTHONDONTWRITEBYTECODE'): - try: - write_code_as_pyc(fpath, code) - except (IOError, OSError): - # We failed to save the bytecode, probably because of a - # permissions issue. The user only asked to import the - # file, so don't bug them about it. - pass - eval(code, module.__dict__) - except (HyTypeError, LexException) as e: - if e.source is None: - with open(fpath, 'rt') as fp: - e.source = fp.read() - e.filename = fpath - raise - except Exception: - sys.modules.pop(module_name, None) - raise - sys.modules[module_name] = module - module.__name__ = module_name - - module.__file__ = os.path.normpath(fpath) - if loader: - module.__loader__ = loader - if is_package(module_name): - module.__path__ = [] - module.__package__ = module_name - else: - module.__package__ = module_name.rpartition('.')[0] - - return module - - -def import_buffer_to_module(module_name, buf): - try: - _ast = import_buffer_to_ast(buf, module_name) - mod = imp.new_module(module_name) - eval(ast_compile(_ast, "", "exec"), mod.__dict__) - except (HyTypeError, LexException) as e: - if e.source is None: - e.source = buf - e.filename = '' - raise - return mod - - -def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): - """``eval`` evaluates a quoted expression and returns the value. The optional - second and third arguments specify the dictionary of globals to use and the - module name. The globals dictionary defaults to ``(local)`` and the module - name defaults to the name of the current module. - - => (eval '(print "Hello World")) - "Hello World" - - If you want to evaluate a string, use ``read-str`` to convert it to a - form first: - - => (eval (read-str "(+ 1 1)")) - 2""" - if namespace is None: - frame = inspect.stack()[1][0] - namespace = inspect.getargvalues(frame).locals - if module_name is None: - m = inspect.getmodule(inspect.stack()[1][0]) - module_name = '__eval__' if m is None else m.__name__ - - foo = HyObject() - foo.start_line = 0 - foo.end_line = 0 - foo.start_column = 0 - foo.end_column = 0 - replace_hy_obj(hytree, foo) - - if not isinstance(module_name, string_types): - raise HyTypeError(foo, "Module name must be a string") - - _ast, expr = hy_compile(hytree, module_name, get_expr=True) - - # Spoof the positions in the generated ast... - for node in ast.walk(_ast): - node.lineno = 1 - node.col_offset = 1 - - for node in ast.walk(expr): - node.lineno = 1 - node.col_offset = 1 - - if ast_callback: - ast_callback(_ast, expr) - - if not isinstance(namespace, dict): - raise HyTypeError(foo, "Globals must be a dictionary") - - # Two-step eval: eval() the body of the exec call - eval(ast_compile(_ast, "", "exec"), namespace) - - # Then eval the expression context and return that - return eval(ast_compile(expr, "", "eval"), namespace) - - -def write_hy_as_pyc(fname): - _ast = import_file_to_ast(fname, - os.path.basename(os.path.splitext(fname)[0])) - code = ast_compile(_ast, fname, "exec") - write_code_as_pyc(fname, code) - - -def write_code_as_pyc(fname, code): - st = os.stat(fname) - timestamp = long_type(st.st_mtime) - - cfile = get_bytecode_path(fname) - try: - os.makedirs(os.path.dirname(cfile)) - except (IOError, OSError): - pass - - with builtins.open(cfile, 'wb') as fc: - fc.write(MAGIC) - if PY37: - # With PEP 552, the header structure has a new flags field - # that we need to fill in. All zeros preserve the legacy - # behaviour, but should we implement reproducible builds, - # this is where we'd add the information. - wr_long(fc, 0) - wr_long(fc, timestamp) - if PY3: - wr_long(fc, st.st_size) - marshal.dump(code, fc) - - -class MetaLoader(object): - def __init__(self, path): - self.path = path - - def load_module(self, fullname): - if fullname in sys.modules: - return sys.modules[fullname] - - if not self.path: - return - - return import_file_to_module(fullname, self.path, self) - - -class MetaImporter(object): - def find_on_path(self, fullname): - fls = ["%s/__init__.hy", "%s.hy"] - dirpath = "/".join(fullname.split(".")) - - for pth in sys.path: - pth = os.path.abspath(pth) - for fp in fls: - composed_path = fp % ("%s/%s" % (pth, dirpath)) - if os.path.exists(composed_path): - return composed_path - - def find_module(self, fullname, path=None): - path = self.find_on_path(fullname) - if path: - return MetaLoader(path) - - -sys.meta_path.insert(0, MetaImporter()) -sys.path.insert(0, "") - - -def is_package(module_name): - mpath = os.path.join(*module_name.split(".")) - for path in map(os.path.abspath, sys.path): - if os.path.exists(os.path.join(path, mpath, "__init__.hy")): - return True - return False - - -def get_bytecode_path(source_path): - if PY3: - import importlib.util - return importlib.util.cache_from_source(source_path) - elif hasattr(imp, "cache_from_source"): - return imp.cache_from_source(source_path) - else: - # If source_path has a file extension, replace it with ".pyc". - # Otherwise, just append ".pyc". - d, f = os.path.split(source_path) - return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f)) diff --git a/hy/importlib/__init__.py b/hy/importlib/__init__.py new file mode 100644 index 000000000..eea974d57 --- /dev/null +++ b/hy/importlib/__init__.py @@ -0,0 +1,116 @@ +# Copyright 2018 the authors. +# This file is part of Hy, which is free software licensed under the Expat +# license. See the LICENSE. + +import ast +import inspect +import pkgutil +import re +import runpy +import __future__ + +from hy.compiler import hy_compile, HyTypeError +from hy.lex import tokenize +from hy.models import HyObject, HyExpression, HySymbol, replace_hy_obj +from hy._compat import PY3, string_types + + +def ast_compile(ast, filename, mode): + """Compile AST. + Like Python's compile, but with some special flags.""" + flags = (__future__.CO_FUTURE_DIVISION | + __future__.CO_FUTURE_PRINT_FUNCTION) + return compile(ast, filename, mode, flags) + + +def hy_parse(source): + source = re.sub(r'\A#!.*', '', source) + return HyExpression([HySymbol("do")] + tokenize(source + "\n")) + + +def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): + """``eval`` evaluates a quoted expression and returns the value. The optional + second and third arguments specify the dictionary of globals to use and the + module name. The globals dictionary defaults to ``(local)`` and the module + name defaults to the name of the current module. + + => (eval '(print "Hello World")) + "Hello World" + + If you want to evaluate a string, use ``read-str`` to convert it to a + form first: + + => (eval (read-str "(+ 1 1)")) + 2""" + if namespace is None: + frame = inspect.stack()[1][0] + namespace = inspect.getargvalues(frame).locals + if module_name is None: + m = inspect.getmodule(inspect.stack()[1][0]) + module_name = '__eval__' if m is None else m.__name__ + + foo = HyObject() + foo.start_line = 0 + foo.end_line = 0 + foo.start_column = 0 + foo.end_column = 0 + replace_hy_obj(hytree, foo) + + if not isinstance(module_name, string_types): + raise HyTypeError(foo, "Module name must be a string") + + _ast, expr = hy_compile(hytree, module_name, get_expr=True) + + # Spoof the positions in the generated ast... + for node in ast.walk(_ast): + node.lineno = 1 + node.col_offset = 1 + + for node in ast.walk(expr): + node.lineno = 1 + node.col_offset = 1 + + if ast_callback: + ast_callback(_ast, expr) + + if not isinstance(namespace, dict): + raise HyTypeError(foo, "Globals must be a dictionary") + + # Two-step eval: eval() the body of the exec call + eval(ast_compile(_ast, "", "exec"), namespace) + + # Then eval the expression context and return that + return eval(ast_compile(expr, "", "eval"), namespace) + + +# Monkeypatch _get_code_from_file so it will try to compile the file +# as Hy code first, then fallback to Python. +def _get_code_from_file(run_name, fname): + # Check for a compiled file first + with open(fname, "rb") as f: + code = pkgutil.read_code(f) + + if code is None: + with open(fname, "rb") as f: + source = f.read() + + try: + # Try to run it as Hy code + ast = hy_parse(source.decode('utf-8')) + code = hy_compile(ast, fname) + code = ast_compile(code, fname, 'exec') + except SyntaxError: + # That didn't work, so try it as normal source code + code = compile(source, fname, 'exec') + + return code, fname + + +if PY3: + from .machinery import HyLoader, _install # NOQA + runpy._get_code_from_file = _get_code_from_file +else: + from .compat import HyLoader, _install # NOQA + runpy._get_code_from_file = lambda f: _get_code_from_file(None, f)[0] + +_install() diff --git a/hy/importlib/bytecode.py b/hy/importlib/bytecode.py new file mode 100644 index 000000000..212fa2240 --- /dev/null +++ b/hy/importlib/bytecode.py @@ -0,0 +1,117 @@ +import os +import sys +import marshal +from hy._compat import PY3 + +from .util import MAGIC_NUMBER, w_long, r_long, _verbose_message + + +BYTECODE_SUFFIX = '.pyc' + +_PYCACHE = '__pycache__' +_OPT = 'opt-' + + +if PY3: + def _import_error(*args, **kwargs): + return ImportError(*args, **kwargs) +else: + def _import_error(*args, **kwargs): + return ImportError(*args) + + +def get_path(path, optimization=None): + head, tail = os.path.split(path) + base, sep, rest = tail.rpartition('.') + + if not PY3: + return os.path.join(head, base + BYTECODE_SUFFIX) + + tag = sys.implementation.cache_tag + if tag is None: + raise NotImplementedError('sys.implementation.cache_tag is None') + almost_filename = ''.join([(base if base else rest), sep, tag]) + if optimization is None: + if sys.flags.optimize == 0: + optimization = '' + else: + optimization = sys.flags.optimize + optimization = str(optimization) + if optimization != '': + if not optimization.isalnum(): + raise ValueError('{!r} is not alphanumeric'.format(optimization)) + almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization) + return os.path.join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIX) + + +def validate_header(data, source_stats=None, name=None, path=None): + exc_details = {} + if path is not None: + exc_details['path'] = path + + magic = data[:4] + raw_timestamp = data[4:8] + raw_size = data[8:12] + + if magic != MAGIC_NUMBER: + message = 'bad python magic number in {!r}: {!r}'.format(name, magic) + _verbose_message('{}', message) + raise _import_error(message, **exc_details) + elif len(raw_timestamp) != 4: + message = 'reached EOF while reading timestamp in {!r}'.format(name) + _verbose_message('{}', message) + raise EOFError(message) + elif PY3 and len(raw_size) != 4: + message = 'reached EOF while reading size of source in {!r}'.format(name) + _verbose_message('{}', message) + raise EOFError(message) + + if source_stats is not None: + try: + source_mtime = int(source_stats['mtime']) + except KeyError: + pass + else: + if r_long(raw_timestamp) != source_mtime: + message = 'bytecode is stale for {!r}'.format(name) + _verbose_message('{}', message) + raise _import_error(message, **exc_details) + + if PY3: + try: + source_size = source_stats['size'] & 0xFFFFFFFF + except KeyError: + pass + else: + if r_long(raw_size) != source_size: + message = 'bytecode is stale for {!r}'.format(name) + _verbose_message('{}', message) + raise _import_error(message, **exc_details) + + return data[12:] if PY3 else data[8:] + + +if PY3: + _code_type = type(validate_header.__code__) +else: + _code_type = type(validate_header.func_code) + + +def load(data, name=None, bytecode_path=None, source_path=None): + """Compile bytecode as returned by validate_header().""" + code = marshal.loads(data) + if isinstance(code, _code_type): + _verbose_message('code object from {!r}', bytecode_path) + return code + else: + raise _import_error('Non-code object in {!r}'.format(bytecode_path), + name=name, path=bytecode_path) + + +def dump(code, mtime=0, source_size=0): + data = bytearray(MAGIC_NUMBER) + data.extend(w_long(mtime)) + if PY3: + data.extend(w_long(source_size)) + data.extend(marshal.dumps(code)) + return data diff --git a/hy/importlib/compat.py b/hy/importlib/compat.py new file mode 100644 index 000000000..a7cb75a37 --- /dev/null +++ b/hy/importlib/compat.py @@ -0,0 +1,110 @@ +# Copyright 2018 the authors. +# This file is part of Hy, which is free software licensed under the Expat +# license. See the LICENSE. + +from io import open +import imp +import os +import sys + +from . import loader +from .util import write_atomic, _verbose_message + + +class HyLoader(loader.HyLoader): + def __init__(self, fullname, path): + self.name = fullname + self.path = path + + def get_filename(self, fullname): + return self.path + + def path_stats(self, path): + st = os.stat(path) + return {'mtime': st.st_mtime, 'size': st.st_size} + + def is_package(self, fullname): + filename = os.path.split(self.path)[1] + filename_base = filename.rsplit('.', 1)[0] + tail_name = fullname.rpartition('.')[2] + return filename_base == '__init__' and tail_name != '__init__' + + def get_data(self, path): + with open(path, 'rb') as f: + return f.read() + + def set_data(self, path, data, mode=0o666): + parent, filename = os.path.split(path) + path_parts = [] + while parent and not os.path.isdir(parent): + parent, part = os.path.split(parent) + path_parts.append(part) + + for part in reversed(path_parts): + parent = os.path.join(parent, part) + try: + os.mkdir(parent) + except FileExistsError: + continue + except OSError as exc: + _verbose_message('could not create {!r}: {!r}', parent, exc) + return + try: + write_atomic(path, data, mode) + _verbose_message('created {!r}', path) + except OSError as exc: + _verbose_message('could not create {!r}: {!r}', path, exc) + + def load_module(self, fullname): + if fullname in sys.modules: + return sys.modules[fullname] + + module = sys.modules[fullname] = imp.new_module(fullname) + name = module.__name__ + try: + code_object = self.get_code(name) + except Exception: + del sys.modules[fullname] + raise + + module.__file__ = self.path + module.__package__ = name + if self.is_package(name): + module_path = os.path.split(module.__file__)[0] + module.__path__ = [module_path] + sys.path_importer_cache[module_path] = HyPathFinder + else: + module.__package__ = module.__package__.rpartition('.')[0] + module.__loader__ = self + + exec(code_object, module.__dict__) + return module + + +class HyPathFinder(object): + @classmethod + def find_module(cls, fullname, path=None): + root_module, _, tail_module = fullname.rpartition('.') + + if not path: + if root_module: + f, filename, description = imp.find_module(root_module) + module = imp.load_module(root_module, f, filename, description) + return imp.find_module(root_module, module.__path__) + else: + path = sys.path + + for pth in path: + base_path = os.path.join(pth, tail_module) + if os.path.isdir(base_path): + full_path = os.path.join(base_path, '__init__.hy') + if os.path.isfile(full_path): + return HyLoader(fullname, full_path) + else: + full_path = os.path.join(pth, tail_module + '.hy') + if os.path.isfile(full_path): + return HyLoader(fullname, full_path) + + +def _install(): + sys.meta_path.append(HyPathFinder) diff --git a/hy/importlib/loader.py b/hy/importlib/loader.py new file mode 100644 index 000000000..38dfb3159 --- /dev/null +++ b/hy/importlib/loader.py @@ -0,0 +1,77 @@ +import sys + +from hy.compiler import hy_compile +from hy.models import HyExpression, HySymbol +from hy.lex import tokenize +from hy._compat import PY3 + +from . import ast_compile, bytecode, hy_parse +from .util import _verbose_message + + +try: + from importlib.machinery import SourceFileLoader +except ImportError: + SourceFileLoader = object + + +class HyLoader(SourceFileLoader): + def source_to_code(self, data, path, _optimize=-1): + ast = hy_compile(hy_parse(data.decode("utf-8")), self.name) + return ast_compile(ast, path, "exec") + + def get_code(self, fullname): + source_path = self.get_filename(fullname) + source_mtime = None + try: + bytecode_path = bytecode.get_path(source_path) + except NotImplementedError: + bytecode_path = None + else: + try: + st = self.path_stats(source_path) + except IOError as e: + pass + else: + source_mtime = int(st['mtime']) + try: + data = self.get_data(bytecode_path) + except (IOError, OSError): + pass + else: + try: + bytes_data = bytecode.validate_header( + data, source_stats=st, name=fullname, + path=bytecode_path + ) + except (ImportError, EOFError) as err: + pass + else: + _verbose_message('{} matches {}', bytecode_path, + source_path) + + # In Python 2, __file__ reflects what's + # loaded. By fixing this up, we'll set the + # bytecode path instead. + # + # Easier to live with the conditional here + # than having two maintain two copies of this + # function... + if not PY3: + self.path = bytecode_path + + return bytecode.load(bytes_data, name=fullname, + bytecode_path=bytecode_path, + source_path=source_path) + + source_bytes = self.get_data(source_path) + code_object = self.source_to_code(source_bytes, source_path) + _verbose_message('code object from {}', source_path) + + if (not sys.dont_write_bytecode and bytecode_path is not None and + source_mtime is not None): + data = bytecode.dump(code_object, source_mtime, len(source_bytes)) + self.set_data(bytecode_path, data) + _verbose_message('wrote {!r}', bytecode_path) + + return code_object diff --git a/hy/importlib/machinery.py b/hy/importlib/machinery.py new file mode 100644 index 000000000..20ed8aae4 --- /dev/null +++ b/hy/importlib/machinery.py @@ -0,0 +1,89 @@ +# Copyright 2018 the authors. +# This file is part of Hy, which is free software licensed under the Expat +# license. See the LICENSE. + +import sys +import os + +from importlib.machinery import FileFinder, PathFinder + +from . import bytecode +from .loader import HyLoader +from .util import _verbose_message + + +SOURCE_SUFFIXES = [".hy"] + +path_importer_cache = {} +path_hooks = [] + + +class HyFileFinder(FileFinder): + def find_spec(self, fullname, target=None): + tail_module = fullname.rpartition('.')[2] + try: + mtime = os.stat(self.path or os.getcwd()).st_mtime + except OSError: + mtime = -1 + if mtime != self._path_mtime: + self._fill_cache() + self._path_mtime = mtime + + if tail_module in self._path_cache: + base_path = os.path.join(self.path, tail_module) + for suffix, loader_class in self._loaders: + init_filename = '__init__' + suffix + full_path = os.path.join(base_path, init_filename) + if os.path.isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, + [base_path], target) + + for suffix, loader_class in self._loaders: + full_path = os.path.join(self.path, tail_module + suffix) + _verbose_message('trying {}', full_path, verbosity=2) + if tail_module + suffix in self._path_cache: + if os.path.isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, + None, target) + + +class HyPathFinder(PathFinder): + @classmethod + def invalidate_caches(cls): + for finder in path_importer_cache.values(): + if hasattr(finder, 'invalidate_caches'): + finder.invalidate_caches() + + @classmethod + def _path_hooks(cls, path): + for hook in path_hooks: + try: + return hook(path) + except ImportError: + continue + + @classmethod + def _path_importer_cache(cls, path): + if path == '': + try: + path = os.getcwd() + except FileNotFoundError: + return None + try: + finder = path_importer_cache[path] + except KeyError: + finder = cls._path_hooks(path) + path_importer_cache[path] = finder + return finder + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + spec = super().find_spec(fullname, path=path, target=target) + if spec: + spec.cached = bytecode.get_path(spec.origin) + return spec + + +def _install(): + path_hooks.append(HyFileFinder.path_hook((HyLoader, SOURCE_SUFFIXES))) + sys.meta_path.insert(0, HyPathFinder) diff --git a/hy/importlib/util.py b/hy/importlib/util.py new file mode 100644 index 000000000..78273e553 --- /dev/null +++ b/hy/importlib/util.py @@ -0,0 +1,56 @@ +from __future__ import print_function + +import io +import os +import struct +import sys + +from hy._compat import PY3 + + +if PY3: + from importlib.util import MAGIC_NUMBER # NOQA + from importlib._bootstrap import _verbose_message # NOQA + _replace = os.replace +else: + from py_compile import MAGIC as MAGIC_NUMBER # NOQA + _replace = os.rename # best effort fallback + + def _verbose_message(message, verbosity=1, *args): + # Python 2 compat for -v PYTHONVERBOSE. Note that the + # structure of the messages are from Python 3, so might not + # match what someone on Python 2 would expect. + if sys.flags.verbose >= verbosity: + if not message.startswith(('#', 'import ')): + message = '# ' + message + print(message.format(*args), file=sys.stderr) + + +def w_long(x): + return struct.pack(" Date: Sun, 4 Mar 2018 16:14:34 -0500 Subject: [PATCH 3/4] Initial draft of documentation --- docs/index.rst | 1 + docs/loader.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 docs/loader.rst diff --git a/docs/index.rst b/docs/index.rst index 4a6cf41af..527ac0158 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,4 +36,5 @@ Contents: language/index extra/index contrib/index + loader hacking diff --git a/docs/loader.rst b/docs/loader.rst new file mode 100644 index 000000000..019c5a3ac --- /dev/null +++ b/docs/loader.rst @@ -0,0 +1,51 @@ +========= +Hy Loader +========= + +.. warning:: + This is incomplete; please consider contributing to the documentation + effort. + + +The Loader +========== + +Hy, by emits Python AST, can be directly parsed, evaluated, and executed. + +Its Python interoptability is powered by PEP 302 which lets Hy hook +into the Python import logic + +This lets Python (and Hy) reply directly in Python's import system and +load both Python and Hy modules. + +The mechanism differs between different Python versions, and worth +highlighting the various corner cases and poops.... + +Python 3 +-------- + +Implements a `meta_path` entry that provides its own `PathFinder` +which then implements its own `FileFinder` which then chains a +`Loader`. + +The three parts provide a `ModuleSpec` which Python is able to then +convert into a module. + +Python 2 +-------- + +There's a simpler system here, with a `meta_path` entry provides an +`Indexer`. The indexer then finds core and returns a loader, which is +directly responsible for loading a module. When we load a module +directly. + +Known Issues +------------ + +* The Hy metaloader must be specified first, and can't be fallen back on +* Valid Hy modules could be confused as valid Python namespace + modules. This means that a Hy module could correctly import, but not + contain any attributes. +* We can't support namespace modules because otherwise valid Python + modules start looking like Hy namespace modules (reverse of the + problem above). From b1b7216e51f341e915061f9f65a73fc48f7d6355 Mon Sep 17 00:00:00 2001 From: Simon Gomizelj Date: Sun, 1 Apr 2018 16:27:14 -0400 Subject: [PATCH 4/4] icommand needs to be execed directly into the loaded namespace We can't reply on hy_eval here as it doesn't properly associate the appropriate file information. Fixes #1395 --- hy/cmdline.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/hy/cmdline.py b/hy/cmdline.py index 14ec44565..1fff781e1 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -17,7 +17,7 @@ import hy from hy.compiler import HyTypeError from hy.completer import completion, Completer -from hy.importlib import hy_eval, hy_parse, hy_compile +from hy.importlib import ast_compile, hy_eval, hy_parse, hy_compile from hy.lex import LexException, PrematureEndOfInput from hy.lex.parser import mangle from hy.macros import macro, require @@ -211,15 +211,20 @@ def run_repl(hr=None, **kwargs): return 0 -def run_icommand(source, **kwargs): - hr = HyREPL(**kwargs) - if os.path.exists(source): - with open(source, "r") as f: +def run_icommand(filename, **kwargs): + namespace = kwargs.pop('locals', {}) + + if os.path.exists(filename): + with open(filename, "r") as f: source = f.read() - filename = source - else: - filename = '' - hr.runsource(source, filename=filename, symbol='single') + + hytree = hy_parse(source) + ast = hy_compile(hytree, "__main__") + code_object = ast_compile(ast, filename, mode="exec") + namespace = {'__name__': '__main__'} + exec(code_object, namespace) + + hr = HyREPL(locals=namespace, **kwargs) return run_repl(hr)