A New importer

conftest.py

@@ -1,15 +1,20 @@
 import pytest
-import hy
+import hy  # NOQA
 import os
 from hy._compat import PY3, PY35, PY36
 
+
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
+
+def pytest_ignore_collect(path, config):
+    return (("py3_only" in path.basename and not PY3)
+            or ("py35_only" in path.basename and not PY35)
+            or ("py36_only" in path.basename and not PY36))
+
+
 def pytest_collect_file(parent, path):
     if (path.ext == ".hy"
-            and NATIVE_TESTS in path.dirname + os.sep
-            and path.basename != "__init__.hy"
-            and not ("py3_only" in path.basename and not PY3)
-            and not ("py35_only" in path.basename and not PY35)
-            and not ("py36_only" in path.basename and not PY36)):
+        and NATIVE_TESTS in path.dirname + os.sep
+        and path.basename != "__init__.hy"):
         return pytest.Module(path, parent)

docs/index.rst

@@ -36,4 +36,5 @@ Contents:
    language/index
    extra/index
    contrib/index
+   loader
    hacking

docs/loader.rst

@@ -0,0 +1,51 @@
+=========
+Hy Loader
+=========
+
+.. warning::
+    This is incomplete; please consider contributing to the documentation
+    effort.
+
+
+The Loader
+==========
+
+Hy, by emits Python AST, can be directly parsed, evaluated, and executed.
+
+Its Python interoptability is powered by PEP 302 which lets Hy hook
+into the Python import logic
+
+This lets Python (and Hy) reply directly in Python's import system and
+load both Python and Hy modules.
+
+The mechanism differs between different Python versions, and worth
+highlighting the various corner cases and poops....
+
+Python 3
+--------
+
+Implements a `meta_path` entry that provides its own `PathFinder`
+which then implements its own `FileFinder` which then chains a
+`Loader`.
+
+The three parts provide a `ModuleSpec` which Python is able to then
+convert into a module.
+
+Python 2
+--------
+
+There's a simpler system here, with a `meta_path` entry provides an
+`Indexer`. The indexer then finds core and returns a loader, which is
+directly responsible for loading a module. When we load a module
+directly.
+
+Known Issues
+------------
+
+* The Hy metaloader must be specified first, and can't be fallen back on
+* Valid Hy modules could be confused as valid Python namespace
+  modules. This means that a Hy module could correctly import, but not
+  contain any attributes.
+* We can't support namespace modules because otherwise valid Python
+  modules start looking like Hy namespace modules (reverse of the
+  problem above).

hy/__init__.py

@@ -8,9 +8,9 @@
 from hy.models import HyExpression, HyInteger, HyKeyword, HyComplex, HyString, HyBytes, HySymbol, HyFloat, HyDict, HyList, HySet, HyCons  # NOQA
 
 
-import hy.importer  # NOQA
+import hy.importlib  # NOQA
 # we import for side-effects.
 
 
 from hy.core.language import read, read_str, mangle, unmangle  # NOQA
-from hy.importer import hy_eval as eval  # NOQA
+from hy.importlib import hy_eval as eval  # NOQA

hy/_compat.py

@@ -6,18 +6,7 @@
     import __builtin__ as builtins
 except ImportError:
     import builtins  # NOQA
-try:
-    from py_compile import MAGIC, wr_long
-except ImportError:
-    # py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4
-    from importlib.util import MAGIC_NUMBER as MAGIC  # NOQA
-
-    def wr_long(f, x):
-        """Internal; write a 32-bit int to a file in little-endian order."""
-        f.write(bytes([x & 0xff,
-                       (x >> 8) & 0xff,
-                       (x >> 16) & 0xff,
-                       (x >> 24) & 0xff]))
+
 import sys, keyword
 
 PY3 = sys.version_info[0] >= 3
@@ -60,3 +49,8 @@ def isidentifier(x):
     except T.TokenError:
         return False
     return len(tokens) == 2 and tokens[0][0] == T.NAME
+
+try:
+    FileNotFoundError = FileNotFoundError
+except NameError:
+    FileNotFoundError = IOError

hy/cmdline.py

@@ -10,26 +10,19 @@
 import sys
 import os
 import importlib
+import runpy
 
 import astor.code_gen
 
 import hy
-
+from hy.compiler import HyTypeError
+from hy.completer import completion, Completer
+from hy.importlib import ast_compile, hy_eval, hy_parse, hy_compile
 from hy.lex import LexException, PrematureEndOfInput
 from hy.lex.parser import mangle
-from hy.compiler import HyTypeError
-from hy.importer import (hy_eval, import_buffer_to_module,
-                         import_file_to_ast, import_file_to_hst,
-                         import_buffer_to_ast, import_buffer_to_hst)
-from hy.completer import completion
-from hy.completer import Completer
-
-from hy.errors import HyIOError
-
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
-
-from hy._compat import builtins, PY3
+from hy._compat import builtins, PY3, FileNotFoundError
 
 
 class HyQuitter(object):
@@ -39,15 +32,14 @@ def __init__(self, name):
     def __repr__(self):
         return "Use (%s) or Ctrl-D (i.e. EOF) to exit" % (self.name)
 
-    __str__ = __repr__
-
     def __call__(self, code=None):
         try:
             sys.stdin.close()
         except:
             pass
         raise SystemExit(code)
 
+
 builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
@@ -77,7 +69,7 @@ def runsource(self, source, filename='<input>', symbol='single'):
         global SIMPLE_TRACEBACKS
         try:
             try:
-                do = import_buffer_to_hst(source)
+                do = hy_parse(source)
             except PrematureEndOfInput:
                 return True
         except LexException as e:
@@ -95,7 +87,7 @@ def ast_callback(main_ast, expr_ast):
                     new_ast = ast.Module(main_ast.body +
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
-            value = hy_eval(do, self.locals, "__console__",
+            value = hy_eval(do, self.locals, "__main__",
                             ast_callback)
         except HyTypeError as e:
             if e.source is None:
@@ -175,7 +167,6 @@ def ideas_macro(ETname):
 
 """)])
 
-require("hy.cmdline", "__console__", all_macros=True)
 require("hy.cmdline", "__main__", all_macros=True)
 
 SIMPLE_TRACEBACKS = True
@@ -192,34 +183,15 @@ def pretty_error(func, *args, **kw):
 
 
 def run_command(source):
-    pretty_error(import_buffer_to_module, "__main__", source)
-    return 0
-
-
-def run_module(mod_name):
-    from hy.importer import MetaImporter
-    pth = MetaImporter().find_on_path(mod_name)
-    if pth is not None:
-        sys.argv = [pth] + sys.argv
-        return run_file(pth)
-
-    print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name),
-          file=sys.stderr)
-    return 1
-
-
-def run_file(filename):
-    from hy.importer import import_file_to_module
-    pretty_error(import_file_to_module, "__main__", filename)
-    return 0
+    hy_eval(hy_parse(source))
 
 
 def run_repl(hr=None, **kwargs):
     import platform
     sys.ps1 = "=> "
     sys.ps2 = "... "
 
-    namespace = {'__name__': '__console__', '__doc__': ''}
+    namespace = {'__name__': '__main__', '__doc__': ''}
 
     with completion(Completer(namespace)):
 
@@ -239,15 +211,20 @@ def run_repl(hr=None, **kwargs):
     return 0
 
 
-def run_icommand(source, **kwargs):
-    hr = HyREPL(**kwargs)
-    if os.path.exists(source):
-        with open(source, "r") as f:
+def run_icommand(filename, **kwargs):
+    namespace = kwargs.pop('locals', {})
+
+    if os.path.exists(filename):
+        with open(filename, "r") as f:
             source = f.read()
-        filename = source
-    else:
-        filename = '<input>'
-    hr.runsource(source, filename=filename, symbol='single')
+
+        hytree = hy_parse(source)
+        ast = hy_compile(hytree, "__main__")
+        code_object = ast_compile(ast, filename, mode="exec")
+        namespace = {'__name__': '__main__'}
+        exec(code_object, namespace)
+
+    hr = HyREPL(locals=namespace, **kwargs)
     return run_repl(hr)
 
 
@@ -316,7 +293,8 @@ def cmdline_handler(scriptname, argv):
 
     if options.mod:
         # User did "hy -m ..."
-        return run_module(options.mod)
+        runpy.run_module(options.mod, run_name='__main__', alter_sys=True)
+        return 0
 
     if options.icommand:
         # User did "hy -i ..."
@@ -331,10 +309,11 @@ def cmdline_handler(scriptname, argv):
         else:
             # User did "hy <filename>"
             try:
-                return run_file(options.args[0])
-            except HyIOError as e:
-                print("hy: Can't open file '{0}': [Errno {1}] {2}\n".format(
-                    e.filename, e.errno, e.strerror), file=sys.stderr)
+                runpy.run_path(options.args[0], run_name='__main__')
+                return 0
+            except FileNotFoundError as e:
+                print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
+                      e.filename, e.errno, e.strerror), file=sys.stderr)
                 sys.exit(e.errno)
 
     # User did NOTHING!
@@ -343,12 +322,12 @@ def cmdline_handler(scriptname, argv):
 
 # entry point for cmd line script "hy"
 def hy_main():
+    sys.path.insert(0, "")
     sys.exit(cmdline_handler("hy", sys.argv))
 
 
 # entry point for cmd line script "hyc"
 def hyc_main():
-    from hy.importer import write_hy_as_pyc
     parser = argparse.ArgumentParser(prog="hyc")
     parser.add_argument("files", metavar="FILE", nargs='+',
                         help="file to compile")
@@ -358,12 +337,14 @@ def hyc_main():
 
     for file in options.files:
         try:
-            print("Compiling %s" % file)
-            pretty_error(write_hy_as_pyc, file)
-        except IOError as x:
-            print("hyc: Can't open file '{0}': [Errno {1}] {2}\n".format(
-                x.filename, x.errno, x.strerror), file=sys.stderr)
-            sys.exit(x.errno)
+            print("Compiling {}".format(file))
+            with open(file):
+                # TODO
+                pass
+        except FileNotFoundError as e:
+            print("hyc: Can't open file '{0}': [Errno {1}] {2}".format(
+                  e.filename, e.errno, e.strerror), file=sys.stderr)
+            sys.exit(e.errno)
 
 
 # entry point for cmd line script "hy2py"
@@ -387,14 +368,14 @@ def hy2py_main():
 
     options = parser.parse_args(sys.argv[1:])
 
-    stdin_text = None
     if options.FILE is None or options.FILE == '-':
-        stdin_text = sys.stdin.read()
+        source = sys.stdin.read()
+    else:
+        with open(options.FILE) as source_file:
+            source = source_file.read()
 
+    hst = (pretty_error(hy_parse, source))
     if options.with_source:
-        hst = (pretty_error(import_file_to_hst, options.FILE)
-               if stdin_text is None
-               else pretty_error(import_buffer_to_hst, stdin_text))
         # need special printing on Windows in case the
         # codepage doesn't support utf-8 characters
         if PY3 and platform.system() == "Windows":
@@ -408,9 +389,7 @@ def hy2py_main():
         print()
         print()
 
-    _ast = (pretty_error(import_file_to_ast, options.FILE, module_name)
-            if stdin_text is None
-            else pretty_error(import_buffer_to_ast, stdin_text, module_name))
+    _ast = (pretty_error(hy_compile, hst, module_name))
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))

hy/compiler.py

@@ -3,6 +3,8 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
+from __future__ import absolute_import
+
 from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
                        HyString, HyBytes, HySymbol, HyFloat, HyList, HySet,
                        HyDict, HyCons, wrap_value)
@@ -15,7 +17,7 @@
     str_type, string_types, bytes_type, long_type, PY3, PY35, PY37,
     raise_empty)
 from hy.macros import require, macroexpand, tag_macroexpand
-import hy.importer
+import hy.importlib
 import hy.inspect
 
 import traceback
@@ -2120,9 +2122,9 @@ def compile_dispatch_tag_macro(self, expression):
     @builds("eval-and-compile", "eval-when-compile")
     def compile_eval_and_compile(self, expression, building):
         expression[0] = HySymbol("do")
-        hy.importer.hy_eval(expression,
-                            compile_time_ns(self.module_name),
-                            self.module_name)
+        hy.importlib.hy_eval(expression,
+                             compile_time_ns(self.module_name),
+                             self.module_name)
         return (self._compile_branch(expression[1:])
                 if building == "eval_and_compile"
                 else Result())

hy/core/language.hy

@@ -22,7 +22,7 @@
 (import [hy.lex [LexException PrematureEndOfInput tokenize]])
 (import [hy.lex.parser [mangle unmangle]])
 (import [hy.compiler [HyASTCompiler spoof-positions]])
-(import [hy.importer [hy-eval :as eval]])
+(import [hy.importlib [hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."