Merge pull request #1672 from brandonwillard/new-patch-importer

New patch importer

NEWS.rst

@@ -15,6 +15,9 @@ New Features
 
 Bug Fixes
 ------------------------------
+* Fixed module reloading.
+* Fixed circular imports.
+* Fixed `__main__` file execution.
 * Fixed bugs in the handling of unpacking forms in method calls and
   attribute access.
 * Fixed crashes on Windows when calling `hy-repr` on date and time

conftest.py

@@ -1,15 +1,50 @@
+import os
+import importlib
+
+import py
 import pytest
 import hy
-import os
 from hy._compat import PY3, PY35, PY36
 
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
+_fspath_pyimport = py.path.local.pyimport
+
+
+def pytest_ignore_collect(path, config):
+    return (("py3_only" in path.basename and not PY3) or
+            ("py35_only" in path.basename and not PY35) or
+            ("py36_only" in path.basename and not PY36) or None)
+
+
+def pyimport_patch_mismatch(self, **kwargs):
+    """Lame fix for https://github.com/pytest-dev/py/issues/195"""
+    try:
+        return _fspath_pyimport(self, **kwargs)
+    except py.path.local.ImportMismatchError:
+        pkgpath = self.pypkgpath()
+        if pkgpath is None:
+            pkgroot = self.dirpath()
+            modname = self.purebasename
+        else:
+            pkgroot = pkgpath.dirpath()
+            names = self.new(ext="").relto(pkgroot).split(self.sep)
+            if names[-1] == "__init__":
+                names.pop()
+            modname = ".".join(names)
+
+        res = importlib.import_module(modname)
+
+        return res
+
+
+py.path.local.pyimport = pyimport_patch_mismatch
+
+
 def pytest_collect_file(parent, path):
     if (path.ext == ".hy"
-            and NATIVE_TESTS in path.dirname + os.sep
+        and NATIVE_TESTS in path.dirname + os.sep
-            and path.basename != "__init__.hy"
+        and path.basename != "__init__.hy"):
-            and not ("py3_only" in path.basename and not PY3)
+
-            and not ("py35_only" in path.basename and not PY35)
+        pytest_mod = pytest.Module(path, parent)
-            and not ("py36_only" in path.basename and not PY36)):
+        return pytest_mod
-        return pytest.Module(path, parent)

docs/contrib/walk.rst

@@ -231,7 +231,7 @@ But assignments via ``import`` are always hoisted to normal Python scope, and
 likewise, ``defclass`` will assign the class to the Python scope,
 even if it shares the name of a let binding.
 
-Use ``__import__`` and ``type`` (or whatever metaclass) instead,
+Use ``importlib.import_module`` and ``type`` (or whatever metaclass) instead,
 if you must avoid this hoisting.
 
 The ``let`` macro takes two parameters: a list defining *variables*

hy/_compat.py

@@ -6,18 +6,6 @@ try:
     import __builtin__ as builtins
 except ImportError:
     import builtins  # NOQA
-try:
-    from py_compile import MAGIC, wr_long
-except ImportError:
-    # py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4
-    from importlib.util import MAGIC_NUMBER as MAGIC  # NOQA
-
-    def wr_long(f, x):
-        """Internal; write a 32-bit int to a file in little-endian order."""
-        f.write(bytes([x & 0xff,
-                       (x >> 8) & 0xff,
-                       (x >> 16) & 0xff,
-                       (x >> 24) & 0xff]))
 import sys, keyword
 
 PY3 = sys.version_info[0] >= 3
@@ -60,3 +48,8 @@ def isidentifier(x):
     except T.TokenError:
         return False
     return len(tokens) == 2 and tokens[0][0] == T.NAME
+
+try:
+    FileNotFoundError = FileNotFoundError
+except NameError:
+    FileNotFoundError = IOError

hy/cmdline.py

@@ -9,26 +9,21 @@ import code
 import ast
 import sys
 import os
+import io
 import importlib
+import py_compile
+import runpy
 
 import astor.code_gen
 
 import hy
-
 from hy.lex import LexException, PrematureEndOfInput, mangle
-from hy.compiler import HyTypeError
+from hy.compiler import HyTypeError, hy_compile
-from hy.importer import (hy_eval, import_buffer_to_module,
+from hy.importer import hy_eval, hy_parse
-                         import_file_to_ast, import_file_to_hst,
+from hy.completer import completion, Completer
-                         import_buffer_to_ast, import_buffer_to_hst)
-from hy.completer import completion
-from hy.completer import Completer
-
-from hy.errors import HyIOError
-
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
-
+from hy._compat import builtins, PY3, FileNotFoundError
-from hy._compat import builtins, PY3
 
 
 class HyQuitter(object):
@@ -47,6 +42,7 @@ class HyQuitter(object):
             pass
         raise SystemExit(code)
 
+
 builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
@@ -88,7 +84,7 @@ class HyREPL(code.InteractiveConsole):
 
         try:
             try:
-                do = import_buffer_to_hst(source)
+                do = hy_parse(source)
             except PrematureEndOfInput:
                 return True
         except LexException as e:
@@ -202,25 +198,8 @@ def pretty_error(func, *args, **kw):
 
 
 def run_command(source):
-    pretty_error(import_buffer_to_module, "__main__", source)
+    tree = hy_parse(source)
-    return 0
+    pretty_error(hy_eval, tree, module_name="__main__")
-
-
-def run_module(mod_name):
-    from hy.importer import MetaImporter
-    pth = MetaImporter().find_on_path(mod_name)
-    if pth is not None:
-        sys.argv = [pth] + sys.argv
-        return run_file(pth)
-
-    print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name),
-          file=sys.stderr)
-    return 1
-
-
-def run_file(filename):
-    from hy.importer import import_file_to_module
-    pretty_error(import_file_to_module, "__main__", filename)
     return 0
 
 
@@ -250,13 +229,21 @@ def run_repl(hr=None, **kwargs):
 
 
 def run_icommand(source, **kwargs):
-    hr = HyREPL(**kwargs)
     if os.path.exists(source):
-        with open(source, "r") as f:
+        # Emulate Python cmdline behavior by setting `sys.path` relative
+        # to the executed file's location.
+        if sys.path[0] == '':
+            sys.path[0] = os.path.realpath(os.path.split(source)[0])
+        else:
+            sys.path.insert(0, os.path.split(source)[0])
+
+        with io.open(source, "r", encoding='utf-8') as f:
             source = f.read()
         filename = source
     else:
         filename = '<input>'
+
+    hr = HyREPL(**kwargs)
     hr.runsource(source, filename=filename, symbol='single')
     return run_repl(hr)
 
@@ -283,6 +270,8 @@ def cmdline_handler(scriptname, argv):
                         help="module to run, passed in as a string")
     parser.add_argument("-E", action='store_true',
                         help="ignore PYTHON* environment variables")
+    parser.add_argument("-B", action='store_true',
+                        help="don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x")
     parser.add_argument("-i", dest="icommand",
                         help="program passed in as a string, then stay in REPL")
     parser.add_argument("--spy", action="store_true",
@@ -299,13 +288,17 @@ def cmdline_handler(scriptname, argv):
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
 
-    # stash the hy executable in case we need it later
+    # Get the path of the Hy cmdline executable and swap it with
-    # mimics Python sys.executable
+    # `sys.executable` (saving the original, just in case).
+    # XXX: The `__main__` module will also have `__file__` set to the
+    # entry-point script.  Currently, I don't see an immediate problem, but
+    # that's not how the Python cmdline works.
     hy.executable = argv[0]
+    hy.sys_executable = sys.executable
+    sys.executable = hy.executable
 
-    # need to split the args if using "-m"
+    # Need to split the args.  If using "-m" all args after the MOD are sent to
-    # all args after the MOD are sent to the module
+    # the module in sys.argv.
-    # in sys.argv
     module_args = []
     if "-m" in argv:
         mloc = argv.index("-m")
@@ -319,20 +312,22 @@ def cmdline_handler(scriptname, argv):
         global SIMPLE_TRACEBACKS
         SIMPLE_TRACEBACKS = False
 
-    # reset sys.argv like Python
-    sys.argv = options.args + module_args or [""]
-
     if options.E:
         # User did "hy -E ..."
         _remove_python_envs()
 
+    if options.B:
+        sys.dont_write_bytecode = True
+
     if options.command:
         # User did "hy -c ..."
         return run_command(options.command)
 
     if options.mod:
         # User did "hy -m ..."
-        return run_module(options.mod)
+        sys.argv = [sys.argv[0]] + options.args + module_args
+        runpy.run_module(options.mod, run_name='__main__', alter_sys=True)
+        return 0
 
     if options.icommand:
         # User did "hy -i ..."
@@ -346,11 +341,22 @@ def cmdline_handler(scriptname, argv):
 
         else:
             # User did "hy <filename>"
+            filename = options.args[0]
+
+            # Emulate Python cmdline behavior by setting `sys.path` relative
+            # to the executed file's location.
+            if sys.path[0] == '':
+                sys.path[0] = os.path.realpath(os.path.split(filename)[0])
+            else:
+                sys.path.insert(0, os.path.split(filename)[0])
+
             try:
-                return run_file(options.args[0])
+                sys.argv = options.args
-            except HyIOError as e:
+                runpy.run_path(filename, run_name='__main__')
-                print("hy: Can't open file '{0}': [Errno {1}] {2}\n".format(
+                return 0
-                    e.filename, e.errno, e.strerror), file=sys.stderr)
+            except FileNotFoundError as e:
+                print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
+                      e.filename, e.errno, e.strerror), file=sys.stderr)
                 sys.exit(e.errno)
 
     # User did NOTHING!
@@ -359,27 +365,45 @@ def cmdline_handler(scriptname, argv):
 
 # entry point for cmd line script "hy"
 def hy_main():
+    sys.path.insert(0, "")
     sys.exit(cmdline_handler("hy", sys.argv))
 
 
-# entry point for cmd line script "hyc"
 def hyc_main():
-    from hy.importer import write_hy_as_pyc
     parser = argparse.ArgumentParser(prog="hyc")
-    parser.add_argument("files", metavar="FILE", nargs='+',
+    parser.add_argument("files", metavar="FILE", nargs='*',
-                        help="file to compile")
+                        help=('File(s) to compile (use STDIN if only'
+                              ' "-" or nothing is provided)'))
     parser.add_argument("-v", action="version", version=VERSION)
 
     options = parser.parse_args(sys.argv[1:])
 
-    for file in options.files:
+    rv = 0
-        try:
+    if len(options.files) == 0 or (
-            print("Compiling %s" % file)
+            len(options.files) == 1 and options.files[0] == '-'):
-            pretty_error(write_hy_as_pyc, file)
+        while True:
-        except IOError as x:
+            filename = sys.stdin.readline()
-            print("hyc: Can't open file '{0}': [Errno {1}] {2}\n".format(
+            if not filename:
-                x.filename, x.errno, x.strerror), file=sys.stderr)
+                break
-            sys.exit(x.errno)
+            filename = filename.rstrip('\n')
+            try:
+                py_compile.compile(filename, doraise=True)
+            except py_compile.PyCompileError as error:
+                rv = 1
+                sys.stderr.write("%s\n" % error.msg)
+            except OSError as error:
+                rv = 1
+                sys.stderr.write("%s\n" % error)
+    else:
+        for filename in options.files:
+            try:
+                print("Compiling %s" % filename)
+                py_compile.compile(filename, doraise=True)
+            except py_compile.PyCompileError as error:
+                # return value to indicate at least one failure
+                rv = 1
+                sys.stderr.write("%s\n" % error.msg)
+    return rv
 
 
 # entry point for cmd line script "hy2py"
@@ -403,14 +427,14 @@ def hy2py_main():
 
     options = parser.parse_args(sys.argv[1:])
 
-    stdin_text = None
     if options.FILE is None or options.FILE == '-':
-        stdin_text = sys.stdin.read()
+        source = sys.stdin.read()
+    else:
+        with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
+            source = source_file.read()
 
+    hst = pretty_error(hy_parse, source)
     if options.with_source:
-        hst = (pretty_error(import_file_to_hst, options.FILE)
-               if stdin_text is None
-               else pretty_error(import_buffer_to_hst, stdin_text))
         # need special printing on Windows in case the
         # codepage doesn't support utf-8 characters
         if PY3 and platform.system() == "Windows":
@@ -424,9 +448,7 @@ def hy2py_main():
         print()
         print()
 
-    _ast = (pretty_error(import_file_to_ast, options.FILE, module_name)
+    _ast = pretty_error(hy_compile, hst, module_name)
-            if stdin_text is None
-            else pretty_error(import_buffer_to_ast, stdin_text, module_name))
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))

hy/compiler.py

@@ -1151,7 +1151,7 @@ class HyASTCompiler(object):
                 ret += node(
                     expr, module=module or None, names=names, level=level)
             else: # root == "require"
-                __import__(module)
+                importlib.import_module(module)
                 require(module, self.module_name,
                         assignments=assignments, prefix=prefix)
 

hy/contrib/walk.hy

@@ -299,7 +299,7 @@ But assignments via `import` are always hoisted to normal Python scope, and
 likewise, `defclass` will assign the class to the Python scope,
 even if it shares the name of a let binding.
 
-Use __import__ and type (or whatever metaclass) instead,
+Use `import_module` and `type` (or whatever metaclass) instead,
 if you must avoid this hoisting.
 
 Function arguments can shadow let bindings in their body,

hy/core/macros.hy

@@ -6,6 +6,7 @@
 ;;; These macros form the hy language
 ;;; They are automatically required in every module, except inside hy.core
 
+(import [importlib [import-module]])
 
 (import [hy.models [HyList HySymbol]])
 
@@ -247,13 +248,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
    Use ``#doc foo`` instead for help with tag macro ``#foo``.
    Use ``(help foo)`` instead for help with runtime objects."
   `(try
-     (help (. (__import__ "hy")
+     (help (. (import-module "hy")
               macros
               _hy_macros
               [__name__]
               ['~symbol]))
      (except [KeyError]
-       (help (. (__import__ "hy")
+       (help (. (import-module "hy")
                 macros
                 _hy_macros
                 [None]
@@ -264,13 +265,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
 
    Gets help for a tag macro function available in this module."
   `(try
-     (help (. (__import__ "hy")
+     (help (. (import-module "hy")
               macros
               _hy_tag
               [__name__]
               ['~symbol]))
      (except [KeyError]
-       (help (. (__import__ "hy")
+       (help (. (import-module "hy")
                 macros
                 _hy_tag
                 [None]

hy/importer.py

@@ -4,158 +4,75 @@
 
 from __future__ import absolute_import
 
-from hy.compiler import hy_compile, HyTypeError
-from hy.models import HyExpression, HySymbol
-from hy.lex import tokenize, LexException
-from hy.errors import HyIOError
-
-from io import open
-import re
-import marshal
-import struct
-import imp
 import sys
+import os
 import ast
 import inspect
-import os
+import pkgutil
+import re
+import io
+import runpy
+import types
+import tempfile
+import importlib
 import __future__
 
-from hy._compat import PY3, PY37, MAGIC, builtins, long_type, wr_long
+from hy.errors import HyTypeError
-from hy._compat import string_types
+from hy.compiler import hy_compile
+from hy.lex import tokenize, LexException
+from hy.models import HyExpression, HySymbol
+from hy._compat import string_types, PY3
+
+
+hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
+                        __future__.CO_FUTURE_PRINT_FUNCTION)
 
 
 def ast_compile(ast, filename, mode):
     """Compile AST.
-    Like Python's compile, but with some special flags."""
+
-    flags = (__future__.CO_FUTURE_DIVISION |
+    Parameters
-             __future__.CO_FUTURE_PRINT_FUNCTION)
+    ----------
-    return compile(ast, filename, mode, flags)
+    ast : instance of `ast.AST`
+
+    filename : str
+        Filename used for run-time error messages
+
+    mode: str
+        `compile` mode parameter
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    return compile(ast, filename, mode, hy_ast_compile_flags)
 
 
-def import_buffer_to_hst(buf):
+def hy_parse(source):
-    """Import content from buf and return a Hy AST."""
+    """Parse a Hy source string.
-    return HyExpression([HySymbol("do")] + tokenize(buf + "\n"))
 
+    Parameters
+    ----------
+    source: string
+        Source code to parse.
 
-def import_file_to_hst(fpath):
+    Returns
-    """Import content from fpath and return a Hy AST."""
+    -------
-    try:
+    out : instance of `types.CodeType`
-        with open(fpath, 'r', encoding='utf-8') as f:
+    """
-            buf = f.read()
+    source = re.sub(r'\A#!.*', '', source)
-        # Strip the shebang line, if there is one.
+    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
-        buf = re.sub(r'\A#!.*', '', buf)
-        return import_buffer_to_hst(buf)
-    except IOError as e:
-        raise HyIOError(e.errno, e.strerror, e.filename)
-
-
-def import_buffer_to_ast(buf, module_name):
-    """ Import content from buf and return a Python AST."""
-    return hy_compile(import_buffer_to_hst(buf), module_name)
-
-
-def import_file_to_ast(fpath, module_name):
-    """Import content from fpath and return a Python AST."""
-    return hy_compile(import_file_to_hst(fpath), module_name)
-
-
-def import_file_to_module(module_name, fpath, loader=None):
-    """Import Hy source from fpath and put it into a Python module.
-
-    If there's an up-to-date byte-compiled version of this module, load that
-    instead. Otherwise, byte-compile the module once we're done loading it, if
-    we can.
-
-    Return the module."""
-
-    module = None
-
-    bytecode_path = get_bytecode_path(fpath)
-    try:
-        source_mtime = int(os.stat(fpath).st_mtime)
-        with open(bytecode_path, 'rb') as bc_f:
-            # The first 4 bytes are the magic number for the version of Python
-            # that compiled this bytecode.
-            bytecode_magic = bc_f.read(4)
-            # Python 3.7 introduced a new flags entry in the header structure.
-            if PY37:
-                bc_f.read(4)
-            # The next 4 bytes, interpreted as a little-endian 32-bit integer,
-            # are the mtime of the corresponding source file.
-            bytecode_mtime, = struct.unpack('<i', bc_f.read(4))
-    except (IOError, OSError):
-        pass
-    else:
-        if bytecode_magic == MAGIC and bytecode_mtime >= source_mtime:
-            # It's a cache hit. Load the byte-compiled version.
-            if PY3:
-                # As of Python 3.6, imp.load_compiled still exists, but it's
-                # deprecated. So let's use SourcelessFileLoader instead.
-                from importlib.machinery import SourcelessFileLoader
-                module = (SourcelessFileLoader(module_name, bytecode_path).
-                          load_module(module_name))
-            else:
-                module = imp.load_compiled(module_name, bytecode_path)
-
-    if not module:
-        # It's a cache miss, so load from source.
-        sys.modules[module_name] = None
-        try:
-            _ast = import_file_to_ast(fpath, module_name)
-            module = imp.new_module(module_name)
-            module.__file__ = os.path.normpath(fpath)
-            code = ast_compile(_ast, fpath, "exec")
-            if not os.environ.get('PYTHONDONTWRITEBYTECODE'):
-                try:
-                    write_code_as_pyc(fpath, code)
-                except (IOError, OSError):
-                    # We failed to save the bytecode, probably because of a
-                    # permissions issue. The user only asked to import the
-                    # file, so don't bug them about it.
-                    pass
-            eval(code, module.__dict__)
-        except (HyTypeError, LexException) as e:
-            if e.source is None:
-                with open(fpath, 'rt') as fp:
-                    e.source = fp.read()
-                e.filename = fpath
-            raise
-        except Exception:
-            sys.modules.pop(module_name, None)
-            raise
-        sys.modules[module_name] = module
-        module.__name__ = module_name
-
-    module.__file__ = os.path.normpath(fpath)
-    if loader:
-        module.__loader__ = loader
-    if is_package(module_name):
-        module.__path__ = []
-        module.__package__ = module_name
-    else:
-        module.__package__ = module_name.rpartition('.')[0]
-
-    return module
-
-
-def import_buffer_to_module(module_name, buf):
-    try:
-        _ast = import_buffer_to_ast(buf, module_name)
-        mod = imp.new_module(module_name)
-        eval(ast_compile(_ast, "", "exec"), mod.__dict__)
-    except (HyTypeError, LexException) as e:
-        if e.source is None:
-            e.source = buf
-            e.filename = '<stdin>'
-        raise
-    return mod
 
 
 def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
-    """``eval`` evaluates a quoted expression and returns the value. The optional
+    """Evaluates a quoted expression and returns the value.
-    second and third arguments specify the dictionary of globals to use and the
+
-    module name. The globals dictionary defaults to ``(local)`` and the module
+    The optional second and third arguments specify the dictionary of globals
-    name defaults to the name of the current module.
+    to use and the module name. The globals dictionary defaults to ``(local)``
+    and the module name defaults to the name of the current module.
+
+    Examples
+    --------
 
        => (eval '(print "Hello World"))
        "Hello World"
@@ -164,7 +81,31 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     form first:
 
        => (eval (read-str "(+ 1 1)"))
-       2"""
+       2
+
+    Parameters
+    ----------
+    hytree: a Hy expression tree
+        Source code to parse.
+
+    namespace: dict, optional
+        Namespace in which to evaluate the Hy tree.  Defaults to the calling
+        frame.
+
+    module_name: str, optional
+        Name of the module to which the Hy tree is assigned.  Defaults to
+        the calling frame's module, if any, and '__eval__' otherwise.
+
+    ast_callback: callable, optional
+        A callback that is passed the Hy compiled tree and resulting
+        expression object, in that order, after compilation but before
+        evaluation.
+
+    Returns
+    -------
+    out : Result of evaluating the Hy compiled tree.
+
+    """
     if namespace is None:
         frame = inspect.stack()[1][0]
         namespace = inspect.getargvalues(frame).locals
@@ -199,89 +140,393 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     return eval(ast_compile(expr, "<eval>", "eval"), namespace)
 
 
-def write_hy_as_pyc(fname):
+def cache_from_source(source_path):
-    _ast = import_file_to_ast(fname,
+    """Get the cached bytecode file name for a given source file name.
-                              os.path.basename(os.path.splitext(fname)[0]))
-    code = ast_compile(_ast, fname, "exec")
-    write_code_as_pyc(fname, code)
 
+    This function's name is set to mirror Python 3.x's
+    `importlib.util.cache_from_source`, which is also used when available.
 
-def write_code_as_pyc(fname, code):
+    Parameters
-    st = os.stat(fname)
+    ----------
-    timestamp = long_type(st.st_mtime)
+    source_path : str
+        Path of the source file
 
-    cfile = get_bytecode_path(fname)
+    Returns
-    try:
+    -------
-        os.makedirs(os.path.dirname(cfile))
+    out : str
-    except (IOError, OSError):
+        Path of the corresponding bytecode file that may--or may
-        pass
+        not--actually exist.
-
+    """
-    with builtins.open(cfile, 'wb') as fc:
-        fc.write(MAGIC)
-        if PY37:
-            # With PEP 552, the header structure has a new flags field
-            # that we need to fill in. All zeros preserve the legacy
-            # behaviour, but should we implement reproducible builds,
-            # this is where we'd add the information.
-            wr_long(fc, 0)
-        wr_long(fc, timestamp)
-        if PY3:
-            wr_long(fc, st.st_size)
-        marshal.dump(code, fc)
-
-
-class MetaLoader(object):
-    def __init__(self, path):
-        self.path = path
-
-    def load_module(self, fullname):
-        if fullname in sys.modules:
-            return sys.modules[fullname]
-
-        if not self.path:
-            return
-
-        return import_file_to_module(fullname, self.path, self)
-
-
-class MetaImporter(object):
-    def find_on_path(self, fullname):
-        fls = ["%s/__init__.hy", "%s.hy"]
-        dirpath = "/".join(fullname.split("."))
-
-        for pth in sys.path:
-            pth = os.path.abspath(pth)
-            for fp in fls:
-                composed_path = fp % ("%s/%s" % (pth, dirpath))
-                if os.path.exists(composed_path):
-                    return composed_path
-
-    def find_module(self, fullname, path=None):
-        path = self.find_on_path(fullname)
-        if path:
-            return MetaLoader(path)
-
-
-sys.meta_path.insert(0, MetaImporter())
-sys.path.insert(0, "")
-
-
-def is_package(module_name):
-    mpath = os.path.join(*module_name.split("."))
-    for path in map(os.path.abspath, sys.path):
-        if os.path.exists(os.path.join(path, mpath, "__init__.hy")):
-            return True
-    return False
-
-
-def get_bytecode_path(source_path):
     if PY3:
-        import importlib.util
         return importlib.util.cache_from_source(source_path)
-    elif hasattr(imp, "cache_from_source"):
-        return imp.cache_from_source(source_path)
     else:
         # If source_path has a file extension, replace it with ".pyc".
         # Otherwise, just append ".pyc".
         d, f = os.path.split(source_path)
         return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
+
+
+def _get_code_from_file(run_name, fname=None):
+    """A patch of `runpy._get_code_from_file` that will also compile Hy
+    code.
+
+    This version will read and cache bytecode for Hy files.  It operates
+    normally otherwise.
+    """
+    if fname is None and run_name is not None:
+        fname = run_name
+
+    if fname.endswith('.hy'):
+        full_fname = os.path.abspath(fname)
+        fname_path, fname_file = os.path.split(full_fname)
+        modname = os.path.splitext(fname_file)[0]
+        sys.path.insert(0, fname_path)
+        try:
+            loader = pkgutil.get_loader(modname)
+            code = loader.get_code(modname)
+        finally:
+            sys.path.pop(0)
+    else:
+        with open(fname, "rb") as f:
+            code = pkgutil.read_code(f)
+        if code is None:
+            with open(fname, "rb") as f:
+                source = f.read().decode('utf-8')
+            code = compile(source, fname, 'exec')
+
+    return (code, fname) if PY3 else code
+
+
+_runpy_get_code_from_file = runpy._get_code_from_file
+runpy._get_code_from_file = _get_code_from_file
+
+if PY3:
+    importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy')
+    _py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code
+
+    def _hy_source_to_code(self, data, path, _optimize=-1):
+        if os.path.isfile(path) and path.endswith('.hy'):
+            source = data.decode("utf-8")
+            try:
+                hy_tree = hy_parse(source)
+                data = hy_compile(hy_tree, self.name)
+            except (HyTypeError, LexException) as e:
+                if e.source is None:
+                    e.source = source
+                    e.filename = path
+                raise
+
+        return _py_source_to_code(self, data, path, _optimize=_optimize)
+
+    importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code
+
+    #  This is actually needed; otherwise, pre-created finders assigned to the
+    #  current dir (i.e. `''`) in `sys.path` will not catch absolute imports of
+    #  directory-local modules!
+    sys.path_importer_cache.clear()
+
+    # Do this one just in case?
+    importlib.invalidate_caches()
+
+    # XXX: These and the 2.7 counterparts below aren't truly cross-compliant.
+    # They're useful for testing, though.
+    HyImporter = importlib.machinery.FileFinder
+    HyLoader = importlib.machinery.SourceFileLoader
+
+else:
+    import imp
+    import py_compile
+    import marshal
+    import struct
+    import traceback
+
+    from pkgutil import ImpImporter, ImpLoader
+
+    class HyLoader(ImpLoader, object):
+        def __init__(self, fullname, filename, fileobj=None, etc=None):
+            """This constructor is designed for some compatibility with
+            SourceFileLoader."""
+            if etc is None and filename is not None:
+                if filename.endswith('.hy'):
+                    etc = ('.hy', 'U', imp.PY_SOURCE)
+                    if fileobj is None:
+                        fileobj = io.open(filename, 'rU', encoding='utf-8')
+
+            super(HyLoader, self).__init__(fullname, fileobj, filename, etc)
+
+        def exec_module(self, module, fullname=None):
+            fullname = self._fix_name(fullname)
+            code = self.get_code(fullname)
+            eval(code, module.__dict__)
+
+        def load_module(self, fullname=None):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            fullname = self._fix_name(fullname)
+            ext_type = self.etc[0]
+            mod_type = self.etc[2]
+            mod = None
+            pkg_path = os.path.join(self.filename, '__init__.hy')
+            if ext_type == '.hy' or (
+                    mod_type == imp.PKG_DIRECTORY and
+                    os.path.isfile(pkg_path)):
+
+                if fullname in sys.modules:
+                    mod = sys.modules[fullname]
+                else:
+                    mod = sys.modules.setdefault(
+                        fullname, imp.new_module(fullname))
+
+                # TODO: Should we set these only when not in `sys.modules`?
+                if mod_type == imp.PKG_DIRECTORY:
+                    mod.__file__ = pkg_path
+                    mod.__path__ = [self.filename]
+                    mod.__package__ = fullname
+                else:
+                    # mod.__path__ = self.filename
+                    mod.__file__ = self.get_filename(fullname)
+                    mod.__package__ = '.'.join(fullname.split('.')[:-1])
+
+                mod.__name__ = fullname
+
+                self.exec_module(mod, fullname=fullname)
+
+            if mod is None:
+                self._reopen()
+                try:
+                    mod = imp.load_module(fullname, self.file, self.filename,
+                                          self.etc)
+                finally:
+                    if self.file:
+                        self.file.close()
+
+            mod.__loader__ = self
+            return mod
+
+        def _reopen(self):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            if self.file and self.file.closed:
+                ext_type = self.etc[0]
+                if ext_type == '.hy':
+                    self.file = io.open(self.filename, 'rU', encoding='utf-8')
+                else:
+                    super(HyLoader, self)._reopen()
+
+        def byte_compile_hy(self, fullname=None):
+            fullname = self._fix_name(fullname)
+            if fullname is None:
+                fullname = self.fullname
+            try:
+                hy_source = self.get_source(fullname)
+                hy_tree = hy_parse(hy_source)
+                hy_ast = hy_compile(hy_tree, fullname)
+
+                code = compile(hy_ast, self.filename, 'exec',
+                               hy_ast_compile_flags)
+            except (HyTypeError, LexException) as e:
+                if e.source is None:
+                    e.source = hy_source
+                    e.filename = self.filename
+                raise
+
+            if not sys.dont_write_bytecode:
+                try:
+                    hyc_compile(code)
+                except IOError:
+                    pass
+            return code
+
+        def get_code(self, fullname=None):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            fullname = self._fix_name(fullname)
+            ext_type = self.etc[0]
+            if ext_type == '.hy':
+                # Looks like we have to manually check for--and update--
+                # the bytecode.
+                t_py = long(os.stat(self.filename).st_mtime)
+                pyc_file = cache_from_source(self.filename)
+                if os.path.isfile(pyc_file):
+                    t_pyc = long(os.stat(pyc_file).st_mtime)
+
+                    if t_pyc is not None and t_pyc >= t_py:
+                        with open(pyc_file, 'rb') as f:
+                            if f.read(4) == imp.get_magic():
+                                t = struct.unpack('<I', f.read(4))[0]
+                                if t == t_py:
+                                    self.code = marshal.load(f)
+
+                if self.code is None:
+                    # There's no existing bytecode, or bytecode timestamp
+                    # is older than the source file's.
+                    self.code = self.byte_compile_hy(fullname)
+
+            if self.code is None:
+              super(HyLoader, self).get_code(fullname=fullname)
+
+            return self.code
+
+        def _get_delegate(self):
+            return HyImporter(self.filename).find_module('__init__')
+
+    class HyImporter(ImpImporter, object):
+        def __init__(self, path=None):
+            # We need to be strict about the types of files this importer will
+            # handle.  To start, if the path is not the current directory in
+            # (represented by '' in `sys.path`), then it must be a supported
+            # file type or a directory.  If it isn't, this importer is not
+            # suitable: throw an exception.
+
+            if path == '' or os.path.isdir(path) or (
+                    os.path.isfile(path) and path.endswith('.hy')):
+                self.path = path
+            else:
+                raise ImportError('Invalid path: {}'.format(path))
+
+        def find_loader(self, fullname):
+            return self.find_module(fullname, path=None)
+
+        def find_module(self, fullname, path=None):
+
+            subname = fullname.split(".")[-1]
+
+            if subname != fullname and self.path is None:
+                return None
+
+            if self.path is None:
+                path = None
+            else:
+                path = [os.path.realpath(self.path)]
+
+            fileobj, file_path, etc = None, None, None
+
+            # The following are excerpts from the later pure Python
+            # implementations of the `imp` module (e.g. in Python 3.6).
+            if path is None:
+                path = sys.path
+
+            for entry in path:
+                if (os.path.isfile(entry) and subname == '__main__' and
+                    entry.endswith('.hy')):
+                    file_path = entry
+                    fileobj = io.open(file_path, 'rU', encoding='utf-8')
+                    etc = ('.hy', 'U', imp.PY_SOURCE)
+                    break
+                else:
+                    file_path = os.path.join(entry, subname)
+                    path_init = os.path.join(file_path, '__init__.hy')
+                    if os.path.isfile(path_init):
+                        fileobj = None
+                        etc = ('', '', imp.PKG_DIRECTORY)
+                        break
+
+                    file_path = file_path + '.hy'
+                    if os.path.isfile(file_path):
+                        fileobj = io.open(file_path, 'rU', encoding='utf-8')
+                        etc = ('.hy', 'U', imp.PY_SOURCE)
+                        break
+            else:
+                try:
+                    fileobj, file_path, etc = imp.find_module(subname, path)
+                except (ImportError, IOError):
+                    return None
+
+            return HyLoader(fullname, file_path, fileobj, etc)
+
+    sys.path_hooks.append(HyImporter)
+    sys.path_importer_cache.clear()
+
+    _py_compile_compile = py_compile.compile
+
+    def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False):
+        """Write a Hy file, or code object, to pyc.
+
+        This is a patched version of Python 2.7's `py_compile.compile`.
+
+        Also, it tries its best to write the bytecode file atomically.
+
+        Parameters
+        ----------
+        file_or_code : str or instance of `types.CodeType`
+            A filename for a Hy or Python source file or its corresponding code
+            object.
+        cfile : str, optional
+            The filename to use for the bytecode file.  If `None`, use the
+            standard bytecode filename determined by `cache_from_source`.
+        dfile : str, optional
+            The filename to use for compile-time errors.
+        doraise : bool, default False
+            If `True` raise compilation exceptions; otherwise, ignore them.
+
+        Returns
+        -------
+        out : str
+            The resulting bytecode file name.  Python 3.x returns this, but
+            Python 2.7 doesn't; this function does for convenience.
+        """
+
+        if isinstance(file_or_code, types.CodeType):
+            codeobject = file_or_code
+            filename = codeobject.co_filename
+        else:
+            filename = file_or_code
+
+            with open(filename, 'rb') as f:
+                source_str = f.read().decode('utf-8')
+
+            try:
+                flags = None
+                if filename.endswith('.hy'):
+                    hy_tree = hy_parse(source_str)
+                    source = hy_compile(hy_tree, '<hyc_compile>')
+                    flags = hy_ast_compile_flags
+
+                codeobject = compile(source, dfile or filename, 'exec', flags)
+            except Exception as err:
+                if isinstance(err, (HyTypeError, LexException)) and err.source is None:
+                    err.source = source_str
+                    err.filename = filename
+
+                py_exc = py_compile.PyCompileError(err.__class__, err,
+                                                   dfile or filename)
+                if doraise:
+                    raise py_exc
+                else:
+                    traceback.print_exc()
+                    return
+
+        timestamp = long(os.stat(filename).st_mtime)
+
+        if cfile is None:
+            cfile = cache_from_source(filename)
+
+        f = tempfile.NamedTemporaryFile('wb', dir=os.path.split(cfile)[0],
+                                        delete=False)
+        try:
+            f.write('\0\0\0\0')
+            f.write(struct.pack('<I', timestamp))
+            f.write(marshal.dumps(codeobject))
+            f.flush()
+            f.seek(0, 0)
+            f.write(imp.get_magic())
+
+            # Make sure it's written to disk.
+            f.flush()
+            os.fsync(f.fileno())
+            f.close()
+
+            # Rename won't replace an existing dest on Windows.
+            if os.name == 'nt' and os.path.isfile(cfile):
+                os.unlink(cfile)
+
+            os.rename(f.name, cfile)
+        except OSError:
+            try:
+                os.unlink(f.name)
+            except OSError:
+                pass
+
+        return cfile
+
+    py_compile.compile = hyc_compile

hy/macros.py

@@ -1,6 +1,10 @@
 # Copyright 2018 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
+import pkgutil
+import importlib
+
+from collections import defaultdict
 
 from hy._compat import PY3
 import hy.inspect
@@ -10,8 +14,6 @@ from hy._compat import str_type
 
 from hy.errors import HyTypeError, HyMacroExpansionError
 
-from collections import defaultdict
-
 CORE_MACROS = [
     "hy.core.bootstrap",
 ]
@@ -120,20 +122,14 @@ def load_macros(module_name):
     Other modules get the macros from CORE_MACROS and EXTRA_MACROS.
 
     """
-
-    def _import(module, module_name=module_name):
-        "__import__ a module, avoiding recursions"
-        if module != module_name:
-            __import__(module)
-
     for module in CORE_MACROS:
-        _import(module)
+        importlib.import_module(module)
 
     if module_name.startswith("hy.core"):
         return
 
     for module in EXTRA_MACROS:
-        _import(module)
+        importlib.import_module(module)
 
 
 def make_empty_fn_copy(fn):
@@ -159,8 +155,8 @@ def make_empty_fn_copy(fn):
 def macroexpand(tree, compiler, once=False):
     """Expand the toplevel macros for the `tree`.
 
-    Load the macros from the given `module_name`, then expand the (top-level)
+    Load the macros from the given `compiler.module_name`, then expand the
-    macros in `tree` until we no longer can.
+    (top-level) macros in `tree` until we no longer can.
 
     """
     load_macros(compiler.module_name)

tests/compilers/test_ast.py

@@ -7,8 +7,7 @@ from __future__ import unicode_literals
 
 from hy import HyString
 from hy.models import HyObject
-from hy.compiler import hy_compile
+from hy.importer import hy_compile, hy_eval, hy_parse
-from hy.importer import hy_eval, import_buffer_to_hst
 from hy.errors import HyCompileError, HyTypeError
 from hy.lex.exceptions import LexException
 from hy._compat import PY3
@@ -16,6 +15,7 @@ from hy._compat import PY3
 import ast
 import pytest
 
+
 def _ast_spotcheck(arg, root, secondary):
     if "." in arg:
         local, full = arg.split(".", 1)
@@ -26,16 +26,16 @@ def _ast_spotcheck(arg, root, secondary):
 
 
 def can_compile(expr):
-    return hy_compile(import_buffer_to_hst(expr), "__main__")
+    return hy_compile(hy_parse(expr), "__main__")
 
 
 def can_eval(expr):
-    return hy_eval(import_buffer_to_hst(expr))
+    return hy_eval(hy_parse(expr))
 
 
 def cant_compile(expr):
     try:
-        hy_compile(import_buffer_to_hst(expr), "__main__")
+        hy_compile(hy_parse(expr), "__main__")
         assert False
     except HyTypeError as e:
         # Anything that can't be compiled should raise a user friendly

tests/importer/test_importer.py

@@ -2,35 +2,69 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
-import hy
-from hy.importer import (import_file_to_module, import_buffer_to_ast,
-                         MetaLoader, get_bytecode_path)
-from hy.errors import HyTypeError
 import os
+import sys
 import ast
+import imp
 import tempfile
+import runpy
+import importlib
+
 from fractions import Fraction
+
 import pytest
 
+import hy
+from hy._compat import bytes_type
+from hy.errors import HyTypeError
+from hy.lex import LexException
+from hy.compiler import hy_compile
+from hy.importer import hy_parse, HyLoader, cache_from_source
+
 
 def test_basics():
     "Make sure the basics of the importer work"
-    import_file_to_module("basic",
+
-                          "tests/resources/importer/basic.hy")
+    assert os.path.isfile('tests/resources/__init__.py')
+    resources_mod = importlib.import_module('tests.resources')
+    assert hasattr(resources_mod, 'kwtest')
+
+    assert os.path.isfile('tests/resources/bin/__init__.hy')
+    bin_mod = importlib.import_module('tests.resources.bin')
+    assert hasattr(bin_mod, '_null_fn_for_import_test')
+
+
+def test_runpy():
+    # XXX: `runpy` won't update cached bytecode!  Don't know if that's
+    # intentional or not.
+
+    basic_ns = runpy.run_path('tests/resources/importer/basic.hy')
+    assert 'square' in basic_ns
+
+    main_ns = runpy.run_path('tests/resources/bin')
+    assert main_ns['visited_main'] == 1
+    del main_ns
+
+    main_ns = runpy.run_module('tests.resources.bin')
+    assert main_ns['visited_main'] == 1
+
+    with pytest.raises(IOError):
+        runpy.run_path('tests/resources/foobarbaz.py')
 
 
 def test_stringer():
-    _ast = import_buffer_to_ast("(defn square [x] (* x x))", '')
+    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '')
+
     assert type(_ast.body[0]) == ast.FunctionDef
 
 
 def test_imports():
     path = os.getcwd() + "/tests/resources/importer/a.hy"
-    testLoader = MetaLoader(path)
+    testLoader = HyLoader("tests.resources.importer.a", path)
 
     def _import_test():
         try:
-            return testLoader.load_module("tests.resources.importer.a")
+            return testLoader.load_module()
         except:
             return "Error"
 
@@ -43,7 +77,7 @@ def test_import_error_reporting():
 
     def _import_error_test():
         try:
-            import_buffer_to_ast("(import \"sys\")", '')
+            _ = hy_compile(hy_parse("(import \"sys\")"), '')
         except HyTypeError:
             return "Error reported"
 
@@ -51,24 +85,28 @@ def test_import_error_reporting():
     assert _import_error_test() is not None
 
 
-@pytest.mark.skipif(os.environ.get('PYTHONDONTWRITEBYTECODE'),
+@pytest.mark.skipif(sys.dont_write_bytecode,
                     reason="Bytecode generation is suppressed")
 def test_import_autocompiles():
     "Test that (import) byte-compiles the module."
 
-    f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False)
+    with tempfile.NamedTemporaryFile(suffix='.hy', delete=True) as f:
-    f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
+        f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
-    f.close()
+        f.flush()
 
-    try:
+        pyc_path = cache_from_source(f.name)
-        os.remove(get_bytecode_path(f.name))
-    except (IOError, OSError):
-        pass
-    import_file_to_module("mymodule", f.name)
-    assert os.path.exists(get_bytecode_path(f.name))
 
-    os.remove(f.name)
+        try:
-    os.remove(get_bytecode_path(f.name))
+            os.remove(pyc_path)
+        except (IOError, OSError):
+            pass
+
+        test_loader = HyLoader("mymodule", f.name).load_module()
+
+        assert hasattr(test_loader, 'pyctest')
+        assert os.path.exists(pyc_path)
+
+        os.remove(pyc_path)
 
 
 def test_eval():
@@ -86,3 +124,145 @@ def test_eval():
         '(if True "this is if true" "this is if false")') == "this is if true"
     assert eval_str('(lfor num (range 100) :if (= (% num 2) 1) (pow num 2))') == [
         pow(num, 2) for num in range(100) if num % 2 == 1]
+
+
+def test_reload():
+    """Copied from CPython's `test_import.py`"""
+
+    def unlink(filename):
+        os.unlink(source)
+        bytecode = cache_from_source(source)
+        if os.path.isfile(bytecode):
+            os.unlink(bytecode)
+
+    TESTFN = 'testfn'
+    source = TESTFN + os.extsep + "hy"
+    with open(source, "w") as f:
+        f.write("(setv a 1)")
+        f.write("(setv b 2)")
+
+    sys.path.insert(0, os.curdir)
+    try:
+        mod = importlib.import_module(TESTFN)
+        assert TESTFN in sys.modules
+        assert mod.a == 1
+        assert mod.b == 2
+
+        # On WinXP, just replacing the .py file wasn't enough to
+        # convince reload() to reparse it.  Maybe the timestamp didn't
+        # move enough.  We force it to get reparsed by removing the
+        # compiled file too.
+        unlink(source)
+
+        # Now damage the module.
+        with open(source, "w") as f:
+            f.write("(setv a 10)")
+            f.write("(setv b (// 20 0))")
+
+        with pytest.raises(ZeroDivisionError):
+            imp.reload(mod)
+
+        # But we still expect the module to be in sys.modules.
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        # We should have replaced a w/ 10, but the old b value should
+        # stick.
+        assert mod.a == 10
+        assert mod.b == 2
+
+        # Now fix the issue and reload the module.
+        unlink(source)
+
+        with open(source, "w") as f:
+            f.write("(setv a 11)")
+            f.write("(setv b (// 20 1))")
+
+        imp.reload(mod)
+
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        assert mod.a == 11
+        assert mod.b == 20
+
+        # Now cause a LexException
+        unlink(source)
+
+        with open(source, "w") as f:
+            f.write("(setv a 11")
+            f.write("(setv b (// 20 1))")
+
+        with pytest.raises(LexException):
+            imp.reload(mod)
+
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        assert mod.a == 11
+        assert mod.b == 20
+
+        # Fix it and retry
+        unlink(source)
+
+        with open(source, "w") as f:
+            f.write("(setv a 12)")
+            f.write("(setv b (// 10 1))")
+
+        imp.reload(mod)
+
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        assert mod.a == 12
+        assert mod.b == 10
+
+    finally:
+        del sys.path[0]
+        unlink(source)
+        del sys.modules[TESTFN]
+
+
+def test_circular():
+    """Test circular imports by creating a temporary file/module that calls a
+    function that imports itself."""
+    sys.path.insert(0, os.path.abspath('tests/resources/importer'))
+    try:
+        mod = runpy.run_module('circular')
+        assert mod['f']() == 1
+    finally:
+        sys.path.pop(0)
+
+
+def test_shadowed_basename():
+    """Make sure Hy loads `.hy` files instead of their `.py` counterparts (.e.g
+    `__init__.py` and `__init__.hy`).
+    """
+    sys.path.insert(0, os.path.realpath('tests/resources/importer'))
+    try:
+        assert os.path.isfile('tests/resources/importer/foo/__init__.hy')
+        assert os.path.isfile('tests/resources/importer/foo/__init__.py')
+        assert os.path.isfile('tests/resources/importer/foo/some_mod.hy')
+        assert os.path.isfile('tests/resources/importer/foo/some_mod.py')
+
+        foo = importlib.import_module('foo')
+        assert foo.__file__.endswith('foo/__init__.hy')
+        assert foo.ext == 'hy'
+        some_mod = importlib.import_module('foo.some_mod')
+        assert some_mod.__file__.endswith('foo/some_mod.hy')
+        assert some_mod.ext == 'hy'
+    finally:
+        sys.path.pop(0)
+
+
+def test_docstring():
+    """Make sure a module's docstring is loaded."""
+    sys.path.insert(0, os.path.realpath('tests/resources/importer'))
+    try:
+        mod = importlib.import_module('docstring')
+        expected_doc = ("This module has a docstring.\n\n"
+                        "It covers multiple lines, too!\n")
+        assert mod.__doc__ == expected_doc
+        assert mod.a == 1
+    finally:
+        sys.path.pop(0)

tests/importer/test_pyc.py

@@ -5,20 +5,21 @@
 import os
 import imp
 import tempfile
-from hy.importer import write_hy_as_pyc, get_bytecode_path
+
+import py_compile
 
 
 def test_pyc():
     """Test pyc compilation."""
-    f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False)
+    with tempfile.NamedTemporaryFile(suffix='.hy') as f:
-    f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
+        f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
-    f.close()
+        f.flush()
 
-    write_hy_as_pyc(f.name)
+        cfile = py_compile.compile(f.name)
-    os.remove(f.name)
 
-    cfile = get_bytecode_path(f.name)
+        assert os.path.exists(cfile)
-    mod = imp.load_compiled('pyc', cfile)
-    os.remove(cfile)
 
-    assert mod.pyctest('Foo') == 'XFooY'
+        mod = imp.load_compiled('pyc', cfile)
+        os.remove(cfile)
+
+        assert mod.pyctest('Foo') == 'XFooY'

tests/native_tests/native_macros.hy

@@ -147,7 +147,7 @@
 (defn test-gensym-in-macros []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (setv g (gensym))
       `(do
@@ -160,8 +160,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   ;; and make sure there is something new that starts with _;G|
@@ -173,7 +173,7 @@
 (defn test-with-gensym []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (with-gensyms [a]
         `(do
@@ -186,8 +186,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;a|") s1))
@@ -197,7 +197,7 @@
 (defn test-defmacro/g! []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro/g! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
@@ -209,8 +209,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;res|") s1))
@@ -220,13 +220,13 @@
   ;; defmacro/g! didn't like numbers initially because they
   ;; don't have a startswith method and blew up during expansion
   (setv macro2 "(defmacro/g! two-point-zero [] `(+ (float 1) 1.0))")
-  (assert (import_buffer_to_ast macro2 "foo")))
+  (assert (hy-compile (hy-parse macro2) "foo")))
 
 (defn test-defmacro! []
   ;; defmacro! must do everything defmacro/g! can
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
@@ -238,8 +238,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;res|") s1))
@@ -249,7 +249,7 @@
   ;; defmacro/g! didn't like numbers initially because they
   ;; don't have a startswith method and blew up during expansion
   (setv macro2 "(defmacro! two-point-zero [] `(+ (float 1) 1.0))")
-  (assert (import_buffer_to_ast macro2 "foo"))
+  (assert (hy-compile (hy-parse macro2) "foo"))
 
   (defmacro! foo! [o!foo] `(do ~g!foo ~g!foo))
   ;; test that o! becomes g!

tests/resources/bin/__main__.hy

@@ -0,0 +1,2 @@
+(print "This is a __main__.hy")
+(setv visited_main True)

tests/resources/importer/circular.hy

@@ -0,0 +1,5 @@
+(setv a 1)
+(defn f []
+  (import circular)
+  circular.a)
+(print (f))

tests/resources/importer/docstring.hy

@@ -0,0 +1,5 @@
+"This module has a docstring.
+
+It covers multiple lines, too!
+"
+(setv a 1)

tests/resources/importer/foo/__init__.hy

@@ -0,0 +1,2 @@
+(print "This is __init__.hy")
+(setv ext "hy")

tests/resources/importer/foo/__init__.py

@@ -0,0 +1,2 @@
+print('This is __init__.py')
+ext = 'py'

tests/resources/importer/foo/some_mod.hy

@@ -0,0 +1,2 @@
+(print "This is test_mod.hy")
+(setv ext "hy")

tests/resources/importer/foo/some_mod.py

@@ -0,0 +1,2 @@
+print('This is test_mod.py')
+ext = 'py'

tests/resources/relative_import.hy

@@ -0,0 +1,3 @@
+(import bin.printenv)
+(import sys)
+(print sys.path)

tests/test_bin.py

@@ -6,13 +6,15 @@
 
 import os
 import re
+import sys
 import shlex
 import subprocess
 
+from hy.importer import cache_from_source
+
 import pytest
 
 from hy._compat import builtins
-from hy.importer import get_bytecode_path
 
 
 hy_dir = os.environ.get('HY_DIR', '')
@@ -212,6 +214,10 @@ def test_bin_hy_icmd_file():
     output, _ = run_cmd("hy -i resources/icmd_test_file.hy", "(ideas)")
     assert "Hy!" in output
 
+    file_relative_path = os.path.realpath(os.path.split('tests/resources/relative_import.hy')[0])
+
+    output, _ = run_cmd("hy -i tests/resources/relative_import.hy None")
+    assert file_relative_path in output
 
 def test_bin_hy_icmd_and_spy():
     output, _ = run_cmd("hy -i \"(+ [] [])\" --spy", "(+ 1 1)")
@@ -231,8 +237,11 @@ def test_bin_hy_file_with_args():
 
 
 def test_bin_hyc():
-    _, err = run_cmd("hyc", expect=2)
+    _, err = run_cmd("hyc", expect=0)
-    assert "usage" in err
+    assert err == ''
+
+    _, err = run_cmd("hyc -", expect=0)
+    assert err == ''
 
     output, _ = run_cmd("hyc -h")
     assert "usage" in output
@@ -240,12 +249,12 @@ def test_bin_hyc():
     path = "tests/resources/argparse_ex.hy"
     output, _ = run_cmd("hyc " + path)
     assert "Compiling" in output
-    assert os.path.exists(get_bytecode_path(path))
+    assert os.path.exists(cache_from_source(path))
-    rm(get_bytecode_path(path))
+    rm(cache_from_source(path))
 
 
 def test_bin_hyc_missing_file():
-    _, err = run_cmd("hyc foobarbaz", expect=2)
+    _, err = run_cmd("hyc foobarbaz", expect=1)
     assert "[Errno 2]" in err
 
 
@@ -281,35 +290,40 @@ def test_bin_hy_no_main():
     assert "This Should Still Work" in output
 
 
-@pytest.mark.parametrize('scenario', [
+@pytest.mark.parametrize('scenario', ["normal", "prevent_by_force",
-    "normal", "prevent_by_force", "prevent_by_env"])
+                                      "prevent_by_env", "prevent_by_option"])
-@pytest.mark.parametrize('cmd_fmt', [
+@pytest.mark.parametrize('cmd_fmt', [['hy', '{fpath}'],
-    'hy {fpath}', 'hy -m {modname}', "hy -c '(import {modname})'"])
+                                     ['hy', '-m', '{modname}'],
+                                     ['hy', '-c', "'(import {modname})'"]])
 def test_bin_hy_byte_compile(scenario, cmd_fmt):
 
     modname = "tests.resources.bin.bytecompile"
     fpath = modname.replace(".", "/") + ".hy"
-    cmd = cmd_fmt.format(**locals())
 
-    rm(get_bytecode_path(fpath))
+    if scenario == 'prevent_by_option':
+        cmd_fmt.insert(1, '-B')
+
+    cmd = ' '.join(cmd_fmt).format(**locals())
+
+    rm(cache_from_source(fpath))
 
     if scenario == "prevent_by_force":
         # Keep Hy from being able to byte-compile the module by
         # creating a directory at the target location.
-        os.mkdir(get_bytecode_path(fpath))
+        os.mkdir(cache_from_source(fpath))
 
     # Whether or not we can byte-compile the module, we should be able
     # to run it.
-    output, _ = run_cmd(cmd, dontwritebytecode=scenario == "prevent_by_env")
+    output, _ = run_cmd(cmd, dontwritebytecode=(scenario == "prevent_by_env"))
     assert "Hello from macro" in output
     assert "The macro returned: boink" in output
 
     if scenario == "normal":
         # That should've byte-compiled the module.
-        assert os.path.exists(get_bytecode_path(fpath))
+        assert os.path.exists(cache_from_source(fpath))
-    elif scenario == "prevent_by_env":
+    elif scenario == "prevent_by_env" or scenario == "prevent_by_option":
         # No byte-compiled version should've been created.
-        assert not os.path.exists(get_bytecode_path(fpath))
+        assert not os.path.exists(cache_from_source(fpath))
 
     # When we run the same command again, and we've byte-compiled the
     # module, the byte-compiled version should be run instead of the
@@ -324,6 +338,32 @@ def test_bin_hy_module_main():
     assert "Hello World" in output
 
 
+def test_bin_hy_module_main_file():
+    output, _ = run_cmd("hy -m tests.resources.bin")
+    assert "This is a __main__.hy" in output
+
+    output, _ = run_cmd("hy -m .tests.resources.bin", expect=1)
+
+
+def test_bin_hy_file_main_file():
+    output, _ = run_cmd("hy tests/resources/bin")
+    assert "This is a __main__.hy" in output
+
+
+def test_bin_hy_file_sys_path():
+    """The test resource `relative_import.hy` will perform an absolute import
+    of a module in its directory: a directory that is not on the `sys.path` of
+    the script executing the module (i.e. `hy`).  We want to make sure that Hy
+    adopts the file's location in `sys.path`, instead of the runner's current
+    dir (e.g. '' in `sys.path`).
+    """
+    file_path, _ = os.path.split('tests/resources/relative_import.hy')
+    file_relative_path = os.path.realpath(file_path)
+
+    output, _ = run_cmd("hy tests/resources/relative_import.hy")
+    assert file_relative_path in output
+
+
 def test_bin_hy_module_main_args():
     output, _ = run_cmd("hy -m tests.resources.bin.main test 123")
     assert "test" in output
@@ -337,3 +377,8 @@ def test_bin_hy_module_main_exitvalue():
 def test_bin_hy_module_no_main():
     output, _ = run_cmd("hy -m tests.resources.bin.nomain")
     assert "This Should Still Work" in output
+
+
+def test_bin_hy_sys_executable():
+    output, _ = run_cmd("hy -c '(do (import sys) (print sys.executable))'")
+    assert output.strip().endswith('/hy')