Merge pull request #1682 from brandonwillard/macro-changes

Macro processing updates and fixes

NEWS.rst

@@ -12,9 +12,15 @@ New Features
 * Keyword objects (not just literal keywords) can be called, as
   shorthand for `(get obj :key)`, and they accept a default value
   as a second argument.
+* Minimal macro expansion namespacing has been implemented.  As a result,
+  external macros no longer have to `require` their own macro dependencies.
+* Macros and tags now reside in module-level `__macros__` and `__tags__`
+  attributes.
 
 Bug Fixes
 ------------------------------
+* `require` now compiles to Python AST.
+* Fixed circular `require`s.
 * Fixed module reloading.
 * Fixed circular imports.
 * Fixed `__main__` file execution.

hy/cmdline.py

@@ -13,6 +13,7 @@ import io
 import importlib
 import py_compile
 import runpy
+import types
 
 import astor.code_gen
 
@@ -47,10 +48,26 @@ builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
 
-class HyREPL(code.InteractiveConsole):
+class HyREPL(code.InteractiveConsole, object):
     def __init__(self, spy=False, output_fn=None, locals=None,
                  filename="<input>"):
 
+        super(HyREPL, self).__init__(locals=locals,
+                                     filename=filename)
+
+        # Create a proper module for this REPL so that we can obtain it easily
+        # (e.g. using `importlib.import_module`).
+        # Also, make sure it's properly introduced to `sys.modules` and
+        # consistently use its namespace as `locals` from here on.
+        module_name = self.locals.get('__name__', '__console__')
+        self.module = sys.modules.setdefault(module_name,
+                                             types.ModuleType(module_name))
+        self.module.__dict__.update(self.locals)
+        self.locals = self.module.__dict__
+
+        # Load cmdline-specific macros.
+        require('hy.cmdline', module_name, assignments='ALL')
+
         self.spy = spy
 
         if output_fn is None:
@@ -65,9 +82,6 @@ class HyREPL(code.InteractiveConsole):
             else:
                 self.output_fn = __builtins__[mangle(output_fn)]
 
-        code.InteractiveConsole.__init__(self, locals=locals,
-                                         filename=filename)
-
         # Pre-mangle symbols for repl recent results: *1, *2, *3
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
@@ -102,8 +116,7 @@ class HyREPL(code.InteractiveConsole):
                     new_ast = ast.Module(main_ast.body +
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
-            value = hy_eval(do, self.locals, "__console__",
+            value = hy_eval(do, self.locals, self.module, ast_callback)
-                            ast_callback)
         except HyTypeError as e:
             if e.source is None:
                 e.source = source
@@ -181,8 +194,6 @@ def ideas_macro(ETname):
 
 """)])
 
-require("hy.cmdline", "__console__", assignments="ALL")
-require("hy.cmdline", "__main__", assignments="ALL")
 
 SIMPLE_TRACEBACKS = True
 
@@ -199,7 +210,8 @@ def pretty_error(func, *args, **kw):
 
 def run_command(source):
     tree = hy_parse(source)
-    pretty_error(hy_eval, tree, module_name="__main__")
+    require("hy.cmdline", "__main__", assignments="ALL")
+    pretty_error(hy_eval, tree, None, importlib.import_module('__main__'))
     return 0
 
 
@@ -208,13 +220,13 @@ def run_repl(hr=None, **kwargs):
     sys.ps1 = "=> "
     sys.ps2 = "... "
 
-    namespace = {'__name__': '__console__', '__doc__': ''}
+    if not hr:
+        hr = HyREPL(**kwargs)
+
+    namespace = hr.locals
 
     with completion(Completer(namespace)):
 
-        if not hr:
-            hr = HyREPL(locals=namespace, **kwargs)
-
         hr.interact("{appname} {version} using "
                     "{py}({build}) {pyversion} on {os}".format(
                         appname=hy.__appname__,
@@ -409,7 +421,6 @@ def hyc_main():
 # entry point for cmd line script "hy2py"
 def hy2py_main():
     import platform
-    module_name = "<STDIN>"
 
     options = dict(prog="hy2py", usage="%(prog)s [options] [FILE]",
                    formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -448,7 +459,7 @@ def hy2py_main():
         print()
         print()
 
-    _ast = pretty_error(hy_compile, hst, module_name)
+    _ast = pretty_error(hy_compile, hst, '__main__')
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))

hy/compiler.py

@@ -13,14 +13,16 @@ from hy.errors import HyCompileError, HyTypeError
 
 from hy.lex import mangle, unmangle
 
-import hy.macros
+from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
-from hy._compat import (
+                        PY35, raise_empty)
-    str_type, bytes_type, long_type, PY3, PY35, raise_empty)
+from hy.macros import require, load_macros, macroexpand, tag_macroexpand
-from hy.macros import require, macroexpand, tag_macroexpand
 import hy.importer
 
 import traceback
 import importlib
+import inspect
+import pkgutil
+import types
 import ast
 import sys
 import copy
@@ -279,32 +281,48 @@ def is_unpack(kind, x):
             and x[0] == "unpack-" + kind)
 
 
-_stdlib = {}
-
-
 class HyASTCompiler(object):
+    """A Hy-to-Python AST compiler"""
 
-    def __init__(self, module_name):
+    def __init__(self, module):
+        """
+        Parameters
+        ----------
+        module: str or types.ModuleType
+            Module in which the Hy tree is evaluated.
+        """
         self.anon_var_count = 0
         self.imports = defaultdict(set)
-        self.module_name = module_name
         self.temp_if = None
+
+        if not inspect.ismodule(module):
+            module = importlib.import_module(module)
+
+        self.module = module
+        self.module_name = module.__name__
+
         self.can_use_stdlib = (
-            not module_name.startswith("hy.core")
+            not self.module_name.startswith("hy.core")
-            or module_name == "hy.core.macros")
+            or self.module_name == "hy.core.macros")
+
+        # Load stdlib macros into the module namespace.
+        load_macros(self.module)
+
+        self._stdlib = {}
+
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
-        if self.can_use_stdlib and not _stdlib:
+        if self.can_use_stdlib:
             # Populate _stdlib.
             import hy.core
-            for module in hy.core.STDLIB:
+            for stdlib_module in hy.core.STDLIB:
-                mod = importlib.import_module(module)
+                mod = importlib.import_module(stdlib_module)
-                for e in map(ast_str, mod.EXPORTS):
+                for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
                     if getattr(mod, e) is not getattr(builtins, e, ''):
                         # Don't bother putting a name in _stdlib if it
                         # points to a builtin with the same name. This
                         # prevents pointless imports.
-                        _stdlib[e] = module
+                        self._stdlib[e] = stdlib_module
 
     def get_anon_var(self):
         self.anon_var_count += 1
@@ -1098,11 +1116,6 @@ class HyASTCompiler(object):
         brackets(SYM, sym(":as"), _symn) |
         brackets(SYM, brackets(many(_symn + maybe(sym(":as") + _symn)))))])
     def compile_import_or_require(self, expr, root, entries):
-        """
-        TODO for `require`: keep track of what we've imported in this run and
-        then "unimport" it after we've completed `thing' so that we don't
-        pollute other envs.
-        """
         ret = Result()
 
         for entry in entries:
@@ -1128,8 +1141,9 @@ class HyASTCompiler(object):
                 else:
                     assignments = [(k, v or k) for k, v in kids]
 
+            ast_module = ast_str(module, piecewise=True)
+
             if root == "import":
-                ast_module = ast_str(module, piecewise=True)
                 module = ast_module.lstrip(".")
                 level = len(ast_module) - len(module)
                 if assignments == "ALL" and prefix == "":
@@ -1150,10 +1164,23 @@ class HyASTCompiler(object):
                         for k, v in assignments]
                 ret += node(
                     expr, module=module or None, names=names, level=level)
-            else: # root == "require"
+
-                importlib.import_module(module)
+            elif require(ast_module, self.module, assignments=assignments,
-                require(module, self.module_name,
+                         prefix=prefix):
-                        assignments=assignments, prefix=prefix)
+                # Actually calling `require` is necessary for macro expansions
+                # occurring during compilation.
+                self.imports['hy.macros'].update([None])
+                # The `require` we're creating in AST is the same as above, but used at
+                # run-time (e.g. when modules are loaded via bytecode).
+                ret += self.compile(HyExpression([
+                    HySymbol('hy.macros.require'),
+                    HyString(ast_module),
+                    HySymbol('None'),
+                    HyKeyword('assignments'),
+                    (HyString("ALL") if assignments == "ALL" else
+                        [[HyString(k), HyString(v)] for k, v in assignments]),
+                    HyKeyword('prefix'),
+                    HyString(prefix)]).replace(expr))
 
         return ret
 
@@ -1484,7 +1511,8 @@ class HyASTCompiler(object):
                 [x for pair in attrs[0] for x in pair]).replace(attrs)))
 
         for e in body:
-            e = self.compile(self._rewire_init(macroexpand(e, self)))
+            e = self.compile(self._rewire_init(
+                macroexpand(e, self.module, self)))
             bodyr += e + e.expr_as_stmt()
 
         return bases + asty.ClassDef(
@@ -1520,20 +1548,16 @@ class HyASTCompiler(object):
         return self.compile(tag_macroexpand(
             HyString(mangle(tag)).replace(tag),
             arg,
-            self))
+            self.module))
-
-    _namespaces = {}
 
     @special(["eval-and-compile", "eval-when-compile"], [many(FORM)])
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
-        if self.module_name not in self._namespaces:
+
-            # Initialize a compile-time namespace for this module.
-            self._namespaces[self.module_name] = {
-                'hy': hy, '__name__': self.module_name}
         hy.importer.hy_eval(new_expr + body,
-                            self._namespaces[self.module_name],
+                            self.module.__dict__,
-                            self.module_name)
+                            self.module)
+
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
                 else Result())
@@ -1541,7 +1565,7 @@ class HyASTCompiler(object):
     @builds_model(HyExpression)
     def compile_expression(self, expr):
         # Perform macro expansions
-        expr = macroexpand(expr, self)
+        expr = macroexpand(expr, self.module, self)
         if not isinstance(expr, HyExpression):
             # Go through compile again if the type changed.
             return self.compile(expr)
@@ -1665,8 +1689,8 @@ class HyASTCompiler(object):
                 attr=ast_str(local),
                 ctx=ast.Load())
 
-        if self.can_use_stdlib and ast_str(symbol) in _stdlib:
+        if self.can_use_stdlib and ast_str(symbol) in self._stdlib:
-            self.imports[_stdlib[ast_str(symbol)]].add(ast_str(symbol))
+            self.imports[self._stdlib[ast_str(symbol)]].add(ast_str(symbol))
 
         return asty.Name(symbol, id=ast_str(symbol), ctx=ast.Load())
 
@@ -1699,20 +1723,41 @@ class HyASTCompiler(object):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
-def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
+def hy_compile(tree, module, root=ast.Module, get_expr=False):
     """
-    Compile a HyObject tree into a Python AST Module.
+    Compile a Hy tree into a Python AST tree.
 
-    If `get_expr` is True, return a tuple (module, last_expression), where
+    Parameters
-    `last_expression` is the.
+    ----------
+    module: str or types.ModuleType
+        Module, or name of the module, in which the Hy tree is evaluated.
+
+    root: ast object, optional (ast.Module)
+        Root object for the Python AST tree.
+
+    get_expr: bool, optional (False)
+        If true, return a tuple with `(root_obj, last_expression)`.
+
+    Returns
+    -------
+    out : A Python AST tree
     """
 
+    if isinstance(module, string_types):
+        if module.startswith('<') and module.endswith('>'):
+            module = types.ModuleType(module)
+        else:
+            module = importlib.import_module(ast_str(module, piecewise=True))
+    if not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+
     tree = wrap_value(tree)
     if not isinstance(tree, HyObject):
         raise HyCompileError("`tree` must be a HyObject or capable of "
                              "being promoted to one")
 
-    compiler = HyASTCompiler(module_name)
+    compiler = HyASTCompiler(module)
     result = compiler.compile(tree)
     expr = result.force_expr
 

hy/completer.py

@@ -39,13 +39,15 @@ class Completer(object):
         self.namespace = namespace
         self.path = [hy.compiler._special_form_compilers,
                      builtins.__dict__,
-                     hy.macros._hy_macros[None],
                      namespace]
-        self.tag_path = [hy.macros._hy_tag[None]]
+
-        if '__name__' in namespace:
+        self.tag_path = []
-            module_name = namespace['__name__']
+
-            self.path.append(hy.macros._hy_macros[module_name])
+        namespace.setdefault('__macros__', {})
-            self.tag_path.append(hy.macros._hy_tag[module_name])
+        namespace.setdefault('__tags__', {})
+
+        self.path.append(namespace['__macros__'])
+        self.tag_path.append(namespace['__tags__'])
 
     def attr_matches(self, text):
         # Borrowed from IPython's completer

hy/contrib/walk.hy

@@ -5,6 +5,7 @@
 
 (import [hy [HyExpression HyDict]]
         [functools [partial]]
+        [importlib [import-module]]
         [collections [OrderedDict]]
         [hy.macros [macroexpand :as mexpand]]
         [hy.compiler [HyASTCompiler]])
@@ -42,9 +43,11 @@
 
 (defn macroexpand-all [form &optional module-name]
   "Recursively performs all possible macroexpansions in form."
-  (setv module-name (or module-name (calling-module-name))
+  (setv module (or (and module-name
+                        (import-module module-name))
+                   (calling-module))
         quote-level [0]
-        ast-compiler (HyASTCompiler module-name))  ; TODO: make nonlocal after dropping Python2
+        ast-compiler (HyASTCompiler module))  ; TODO: make nonlocal after dropping Python2
   (defn traverse [form]
     (walk expand identity form))
   (defn expand [form]
@@ -68,7 +71,7 @@
               [(= (first form) (HySymbol "require"))
                (ast-compiler.compile form)
                (return)]
-              [True (traverse (mexpand form ast-compiler))])
+              [True (traverse (mexpand form module ast-compiler))])
         (if (coll? form)
             (traverse form)
             form)))

hy/core/language.hy

@@ -21,7 +21,7 @@
 (import [hy.models [HySymbol HyKeyword]])
 (import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
 (import [hy.compiler [HyASTCompiler]])
-(import [hy.importer [hy-eval :as eval]])
+(import [hy.importer [calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
@@ -295,12 +295,14 @@ Return series of accumulated sums (or other binary function results)."
 (defn macroexpand [form]
   "Return the full macro expansion of `form`."
   (import hy.macros)
-  (hy.macros.macroexpand form (HyASTCompiler (calling-module-name))))
+  (setv module (calling-module))
+  (hy.macros.macroexpand form module (HyASTCompiler module)))
 
 (defn macroexpand-1 [form]
   "Return the single step macro expansion of `form`."
   (import hy.macros)
-  (hy.macros.macroexpand-1 form (HyASTCompiler (calling-module-name))))
+  (setv module (calling-module))
+  (hy.macros.macroexpand-1 form module (HyASTCompiler module)))
 
 (defn merge-with [f &rest maps]
   "Return the map of `maps` joined onto the first via the function `f`.
@@ -467,8 +469,8 @@ Even objects with the __name__ magic will work."
     (or a b)))
 
 (setv EXPORTS
-  '[*map accumulate butlast calling-module-name chain coll? combinations
+  '[*map accumulate butlast calling-module calling-module-name chain coll?
-    comp complement compress constantly count cycle dec distinct
+    combinations comp complement compress constantly count cycle dec distinct
     disassemble drop drop-last drop-while empty? eval even? every? exec first
     filter flatten float? fraction gensym group-by identity inc input instance?
     integer integer? integer-char? interleave interpose islice iterable?

hy/core/macros.hy

@@ -245,34 +245,10 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
 
    Use ``#doc foo`` instead for help with tag macro ``#foo``.
    Use ``(help foo)`` instead for help with runtime objects."
-  `(try
+  `(help (.get __macros__ '~symbol None)))
-     (import [importlib [import-module]])
-     (help (. (import-module "hy")
-              macros
-              _hy_macros
-              [__name__]
-              ['~symbol]))
-     (except [KeyError]
-       (help (. (import-module "hy")
-                macros
-                _hy_macros
-                [None]
-                ['~symbol])))))
 
 (deftag doc [symbol]
   "tag macro documentation
 
    Gets help for a tag macro function available in this module."
-  `(try
+  `(help (.get __tags__ '~symbol None)))
-     (import [importlib [import-module]])
-     (help (. (import-module "hy")
-              macros
-              _hy_tag
-              [__name__]
-              ['~symbol]))
-     (except [KeyError]
-       (help (. (import-module "hy")
-                macros
-                _hy_tag
-                [None]
-                ['~symbol])))))

hy/extra/reserved.hy

@@ -16,7 +16,7 @@
     (setv _cache (frozenset (map unmangle (+
       hy.core.language.EXPORTS
       hy.core.shadow.EXPORTS
-      (list (.keys (get hy.macros._hy_macros None)))
+      (list (.keys hy.core.macros.__macros__))
       keyword.kwlist
       (list (.keys hy.compiler._special_form_compilers))
       (list hy.compiler._bad_roots)))))))

hy/importer.py

@@ -17,9 +17,10 @@ import importlib
 import __future__
 
 from functools import partial
+from contextlib import contextmanager
 
 from hy.errors import HyTypeError
-from hy.compiler import hy_compile
+from hy.compiler import hy_compile, ast_str
 from hy.lex import tokenize, LexException
 from hy.models import HyExpression, HySymbol
 from hy._compat import string_types, PY3
@@ -29,6 +30,36 @@ hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
                         __future__.CO_FUTURE_PRINT_FUNCTION)
 
 
+def calling_module(n=1):
+    """Get the module calling, if available.
+
+    As a fallback, this will import a module using the calling frame's
+    globals value of `__name__`.
+
+    Parameters
+    ----------
+    n: int, optional
+        The number of levels up the stack from this function call.
+        The default is one level up.
+
+    Returns
+    -------
+    out: types.ModuleType
+        The module at stack level `n + 1` or `None`.
+    """
+    frame_up = inspect.stack(0)[n + 1][0]
+    module = inspect.getmodule(frame_up)
+    if module is None:
+        # This works for modules like `__main__`
+        module_name = frame_up.f_globals.get('__name__', None)
+        if module_name:
+            try:
+                module = importlib.import_module(module_name)
+            except ImportError:
+                pass
+    return module
+
+
 def ast_compile(ast, filename, mode):
     """Compile AST.
 
@@ -65,13 +96,9 @@ def hy_parse(source):
     return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
 
 
-def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
+def hy_eval(hytree, locals=None, module=None, ast_callback=None):
     """Evaluates a quoted expression and returns the value.
 
-    The optional second and third arguments specify the dictionary of globals
-    to use and the module name. The globals dictionary defaults to ``(local)``
-    and the module name defaults to the name of the current module.
-
     Examples
     --------
 
@@ -89,13 +116,15 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     hytree: a Hy expression tree
         Source code to parse.
 
-    namespace: dict, optional
+    locals: dict, optional
-        Namespace in which to evaluate the Hy tree.  Defaults to the calling
+        Local environment in which to evaluate the Hy tree.  Defaults to the
-        frame.
+        calling frame.
 
-    module_name: str, optional
+    module: str or types.ModuleType, optional
-        Name of the module to which the Hy tree is assigned.  Defaults to
+        Module, or name of the module, to which the Hy tree is assigned and
-        the calling frame's module, if any, and '__eval__' otherwise.
+        the global values are taken.
+        Defaults to the calling frame's module, if any, and '__eval__'
+        otherwise.
 
     ast_callback: callable, optional
         A callback that is passed the Hy compiled tree and resulting
@@ -105,19 +134,23 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     Returns
     -------
     out : Result of evaluating the Hy compiled tree.
-
     """
-    if namespace is None:
+    if module is None:
+        module = calling_module()
+
+    if isinstance(module, string_types):
+        module = importlib.import_module(ast_str(module, piecewise=True))
+    elif not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    if locals is None:
         frame = inspect.stack()[1][0]
-        namespace = inspect.getargvalues(frame).locals
+        locals = inspect.getargvalues(frame).locals
-    if module_name is None:
-        m = inspect.getmodule(inspect.stack()[1][0])
-        module_name = '__eval__' if m is None else m.__name__
 
-    if not isinstance(module_name, string_types):
+    if not isinstance(locals, dict):
-        raise TypeError("Module name must be a string")
+        raise TypeError("Locals must be a dictionary")
 
-    _ast, expr = hy_compile(hytree, module_name, get_expr=True)
+    _ast, expr = hy_compile(hytree, module, get_expr=True)
 
     # Spoof the positions in the generated ast...
     for node in ast.walk(_ast):
@@ -131,14 +164,13 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     if ast_callback:
         ast_callback(_ast, expr)
 
-    if not isinstance(namespace, dict):
+    globals = module.__dict__
-        raise TypeError("Globals must be a dictionary")
 
     # Two-step eval: eval() the body of the exec call
-    eval(ast_compile(_ast, "<eval_body>", "exec"), namespace)
+    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
 
     # Then eval the expression context and return that
-    return eval(ast_compile(expr, "<eval>", "eval"), namespace)
+    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
 
 
 def cache_from_source(source_path):
@@ -167,6 +199,52 @@ def cache_from_source(source_path):
         return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
 
 
+@contextmanager
+def loader_module_obj(loader):
+    """Use the module object associated with a loader.
+
+    This is intended to be used by a loader object itself, and primarily as a
+    work-around for attempts to get module and/or file code from a loader
+    without actually creating a module object.  Since Hy currently needs the
+    module object for macro importing, expansion, and whatnot, using this will
+    reconcile Hy with such attempts.
+
+    For example, if we're first compiling a Hy script starting from
+    `runpy.run_path`, the Hy compiler will need a valid module object in which
+    to run, but, given the way `runpy.run_path` works, there might not be one
+    yet (e.g. `__main__` for a .hy file).  We compensate by properly loading
+    the module here.
+
+    The function `inspect.getmodule` has a hidden-ish feature that returns
+    modules using their associated filenames (via `inspect.modulesbyfile`),
+    and, since the Loaders (and their delegate Loaders) carry a filename/path
+    associated with the parent package, we use it as a more robust attempt to
+    obtain an existing module object.
+
+    When no module object is found, a temporary, minimally sufficient module
+    object is created for the duration of the `with` body.
+    """
+    tmp_mod = False
+
+    try:
+        module = inspect.getmodule(None, _filename=loader.path)
+    except KeyError:
+        module = None
+
+    if module is None:
+        tmp_mod = True
+        module = sys.modules.setdefault(loader.name,
+                                        types.ModuleType(loader.name))
+        module.__file__ = loader.path
+        module.__name__ = loader.name
+
+    try:
+        yield module
+    finally:
+        if tmp_mod:
+            del sys.modules[loader.name]
+
+
 def _hy_code_from_file(filename, loader_type=None):
     """Use PEP-302 loader to produce code for a given Hy source file."""
     full_fname = os.path.abspath(filename)
@@ -226,7 +304,8 @@ if PY3:
             source = data.decode("utf-8")
             try:
                 hy_tree = hy_parse(source)
-                data = hy_compile(hy_tree, self.name)
+                with loader_module_obj(self) as module:
+                    data = hy_compile(hy_tree, module)
             except (HyTypeError, LexException) as e:
                 if e.source is None:
                     e.source = source
@@ -276,6 +355,15 @@ else:
 
             super(HyLoader, self).__init__(fullname, fileobj, filename, etc)
 
+        def __getattr__(self, item):
+            # We add these for Python >= 3.4 Loader interface compatibility.
+            if item == 'path':
+                return self.filename
+            elif item == 'name':
+                return self.fullname
+            else:
+                return super(HyLoader, self).__getattr__(item)
+
         def exec_module(self, module, fullname=None):
             fullname = self._fix_name(fullname)
             code = self.get_code(fullname)
@@ -283,7 +371,7 @@ else:
 
         def load_module(self, fullname=None):
             """Same as `pkgutil.ImpLoader`, with an extra check for Hy
-            source"""
+            source and the option to not run `self.exec_module`."""
             fullname = self._fix_name(fullname)
             ext_type = self.etc[0]
             mod_type = self.etc[2]
@@ -298,7 +386,7 @@ else:
                     mod = sys.modules[fullname]
                 else:
                     mod = sys.modules.setdefault(
-                        fullname, imp.new_module(fullname))
+                        fullname, types.ModuleType(fullname))
 
                 # TODO: Should we set these only when not in `sys.modules`?
                 if mod_type == imp.PKG_DIRECTORY:
@@ -351,7 +439,8 @@ else:
             try:
                 hy_source = self.get_source(fullname)
                 hy_tree = hy_parse(hy_source)
-                hy_ast = hy_compile(hy_tree, fullname)
+                with loader_module_obj(self) as module:
+                    hy_ast = hy_compile(hy_tree, module)
 
                 code = compile(hy_ast, self.filename, 'exec',
                                hy_ast_compile_flags)
@@ -363,7 +452,7 @@ else:
 
             if not sys.dont_write_bytecode:
                 try:
-                    hyc_compile(code)
+                    hyc_compile(code, module=fullname)
                 except IOError:
                     pass
             return code
@@ -470,7 +559,8 @@ else:
 
     _py_compile_compile = py_compile.compile
 
-    def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False):
+    def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False,
+                    module=None):
         """Write a Hy file, or code object, to pyc.
 
         This is a patched version of Python 2.7's `py_compile.compile`.
@@ -489,6 +579,9 @@ else:
             The filename to use for compile-time errors.
         doraise : bool, default False
             If `True` raise compilation exceptions; otherwise, ignore them.
+        module : str or types.ModuleType, optional
+            The module, or module name, in which the Hy tree is expanded.
+            Default is the caller's module.
 
         Returns
         -------
@@ -510,7 +603,13 @@ else:
                 flags = None
                 if _could_be_hy_src(filename):
                     hy_tree = hy_parse(source_str)
-                    source = hy_compile(hy_tree, '<hyc_compile>')
+
+                    if module is None:
+                        module = inspect.getmodule(inspect.stack()[1][0])
+                    elif not inspect.ismodule(module):
+                        module = importlib.import_module(module)
+
+                    source = hy_compile(hy_tree, module)
                     flags = hy_ast_compile_flags
 
                 codeobject = compile(source, dfile or filename, 'exec', flags)

hy/inspect.py

@@ -1,37 +0,0 @@
-# Copyright 2018 the authors.
-# This file is part of Hy, which is free software licensed under the Expat
-# license. See the LICENSE.
-
-from __future__ import absolute_import
-
-import inspect
-
-try:
-    # Check if we have the newer inspect.signature available.
-    # Otherwise fallback to the legacy getargspec.
-    inspect.signature  # noqa
-except AttributeError:
-    def get_arity(fn):
-        return len(inspect.getargspec(fn)[0])
-
-    def has_kwargs(fn):
-        argspec = inspect.getargspec(fn)
-        return argspec.keywords is not None
-
-    def format_args(fn):
-        argspec = inspect.getargspec(fn)
-        return inspect.formatargspec(*argspec)
-
-else:
-    def get_arity(fn):
-        parameters = inspect.signature(fn).parameters
-        return sum(1 for param in parameters.values()
-                   if param.kind == param.POSITIONAL_OR_KEYWORD)
-
-    def has_kwargs(fn):
-        parameters = inspect.signature(fn).parameters
-        return any(param.kind == param.VAR_KEYWORD
-                   for param in parameters.values())
-
-    def format_args(fn):
-        return str(inspect.signature(fn))

hy/macros.py

@@ -1,19 +1,39 @@
 # Copyright 2018 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
-import pkgutil
 import importlib
+import inspect
+import pkgutil
 
-from collections import defaultdict
+from hy._compat import PY3, string_types
-
-from hy._compat import PY3
-import hy.inspect
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
-from hy._compat import str_type
 
 from hy.errors import HyTypeError, HyMacroExpansionError
 
+try:
+    # Check if we have the newer inspect.signature available.
+    # Otherwise fallback to the legacy getargspec.
+    inspect.signature  # noqa
+except AttributeError:
+    def has_kwargs(fn):
+        argspec = inspect.getargspec(fn)
+        return argspec.keywords is not None
+
+    def format_args(fn):
+        argspec = inspect.getargspec(fn)
+        return inspect.formatargspec(*argspec)
+
+else:
+    def has_kwargs(fn):
+        parameters = inspect.signature(fn).parameters
+        return any(param.kind == param.VAR_KEYWORD
+                   for param in parameters.values())
+
+    def format_args(fn):
+        return str(inspect.signature(fn))
+
+
 CORE_MACROS = [
     "hy.core.bootstrap",
 ]
@@ -22,114 +42,204 @@ EXTRA_MACROS = [
     "hy.core.macros",
 ]
 
-_hy_macros = defaultdict(dict)
-_hy_tag = defaultdict(dict)
-
 
 def macro(name):
     """Decorator to define a macro called `name`.
-
-    This stores the macro `name` in the namespace for the module where it is
-    defined.
-
-    If the module where it is defined is in `hy.core`, then the macro is stored
-    in the default `None` namespace.
-
-    This function is called from the `defmacro` special form in the compiler.
-
     """
     name = mangle(name)
     def _(fn):
         fn.__name__ = '({})'.format(name)
         try:
-            fn._hy_macro_pass_compiler = hy.inspect.has_kwargs(fn)
+            fn._hy_macro_pass_compiler = has_kwargs(fn)
         except Exception:
             # An exception might be raised if fn has arguments with
             # names that are invalid in Python.
             fn._hy_macro_pass_compiler = False
 
-        module_name = fn.__module__
+        module = inspect.getmodule(fn)
-        if module_name.startswith("hy.core"):
+        module_macros = module.__dict__.setdefault('__macros__', {})
-            module_name = None
+        module_macros[name] = fn
-        _hy_macros[module_name][name] = fn
+
         return fn
     return _
 
 
 def tag(name):
     """Decorator to define a tag macro called `name`.
-
-    This stores the macro `name` in the namespace for the module where it is
-    defined.
-
-    If the module where it is defined is in `hy.core`, then the macro is stored
-    in the default `None` namespace.
-
-    This function is called from the `deftag` special form in the compiler.
-
     """
     def _(fn):
         _name = mangle('#{}'.format(name))
+
         if not PY3:
             _name = _name.encode('UTF-8')
+
         fn.__name__ = _name
-        module_name = fn.__module__
+
+        module = inspect.getmodule(fn)
+
+        module_name = module.__name__
         if module_name.startswith("hy.core"):
             module_name = None
-        _hy_tag[module_name][mangle(name)] = fn
+
+        module_tags = module.__dict__.setdefault('__tags__', {})
+        module_tags[mangle(name)] = fn
 
         return fn
     return _
 
 
+def _same_modules(source_module, target_module):
+    """Compare the filenames associated with the given modules names.
+
+    This tries to not actually load the modules.
+    """
+    if not (source_module or target_module):
+        return False
+
+    if target_module == source_module:
+        return True
+
+    def _get_filename(module):
+        filename = None
+        try:
+            if not inspect.ismodule(module):
+                loader = pkgutil.get_loader(module)
+                if loader:
+                    filename = loader.get_filename()
+            else:
+                filename = inspect.getfile(module)
+        except (TypeError, ImportError):
+            pass
+
+        return filename
+
+    source_filename = _get_filename(source_module)
+    target_filename = _get_filename(target_module)
+
+    return (source_filename and target_filename and
+            source_filename == target_filename)
+
+
 def require(source_module, target_module, assignments, prefix=""):
-    """Load macros from `source_module` in the namespace of
+    """Load macros from one module into the namespace of another.
-    `target_module`. `assignments` maps old names to new names, or
-    should be the string "ALL". If `prefix` is nonempty, it is
-    prepended to the name of each imported macro. (This means you get
-    macros named things like "mymacromodule.mymacro", which looks like
-    an attribute of a module, although it's actually just a symbol
-    with a period in its name.)
 
     This function is called from the `require` special form in the compiler.
 
+    Parameters
+    ----------
+    source_module: str or types.ModuleType
+        The module from which macros are to be imported.
+
+    target_module: str, types.ModuleType or None
+        The module into which the macros will be loaded.  If `None`, then
+        the caller's namespace.
+        The latter is useful during evaluation of generated AST/bytecode.
+
+    assignments: str or list of tuples of strs
+        The string "ALL" or a list of macro name and alias pairs.
+
+    prefix: str, optional ("")
+        If nonempty, its value is prepended to the name of each imported macro.
+        This allows one to emulate namespaced macros, like
+        "mymacromodule.mymacro", which looks like an attribute of a module.
+
+    Returns
+    -------
+    out: boolean
+        Whether or not macros and tags were actually transferred.
     """
 
-    seen_names = set()
+    if target_module is None:
+        parent_frame = inspect.stack()[1][0]
+        target_namespace = parent_frame.f_globals
+        target_module = target_namespace.get('__name__', None)
+    elif isinstance(target_module, string_types):
+        target_module = importlib.import_module(target_module)
+        target_namespace = target_module.__dict__
+    elif inspect.ismodule(target_module):
+        target_namespace = target_module.__dict__
+    else:
+        raise TypeError('`target_module` is not a recognized type: {}'.format(
+            type(target_module)))
+
+    # Let's do a quick check to make sure the source module isn't actually
+    # the module being compiled (e.g. when `runpy` executes a module's code
+    # in `__main__`).
+    # We use the module's underlying filename for this (when they exist), since
+    # it's the most "fixed" attribute.
+    if _same_modules(source_module, target_module):
+        return False
+
+    if not inspect.ismodule(source_module):
+        source_module = importlib.import_module(source_module)
+
+    source_macros = source_module.__dict__.setdefault('__macros__', {})
+    source_tags = source_module.__dict__.setdefault('__tags__', {})
+
+    if len(source_module.__macros__) + len(source_module.__tags__) == 0:
+        if assignments != "ALL":
+            raise ImportError('The module {} has no macros or tags'.format(
+                source_module))
+        else:
+            return False
+
+    target_macros = target_namespace.setdefault('__macros__', {})
+    target_tags = target_namespace.setdefault('__tags__', {})
+
     if prefix:
         prefix += "."
-    if assignments != "ALL":
-        assignments = {mangle(str_type(k)): v for k, v in assignments}
 
-    for d in _hy_macros, _hy_tag:
+    if assignments == "ALL":
-        for name, macro in d[source_module].items():
+        # Only add macros/tags created in/by the source module.
-            seen_names.add(name)
+        name_assigns = [(n, n) for n, f in source_macros.items()
-            if assignments == "ALL":
+                        if inspect.getmodule(f) == source_module]
-                d[target_module][mangle(prefix + name)] = macro
+        name_assigns += [(n, n) for n, f in source_tags.items()
-            elif name in assignments:
+                         if inspect.getmodule(f) == source_module]
-                d[target_module][mangle(prefix + assignments[name])] = macro
+    else:
+        # If one specifically requests a macro/tag not created in the source
+        # module, I guess we allow it?
+        name_assigns = assignments
 
-    if assignments != "ALL":
+    for name, alias in name_assigns:
-        unseen = frozenset(assignments.keys()).difference(seen_names)
+        _name = mangle(name)
-        if unseen:
+        alias = mangle(prefix + alias)
-            raise ImportError("cannot require names: " + repr(list(unseen)))
+        if _name in source_module.__macros__:
+            target_macros[alias] = source_macros[_name]
+        elif _name in source_module.__tags__:
+            target_tags[alias] = source_tags[_name]
+        else:
+            raise ImportError('Could not require name {} from {}'.format(
+                _name, source_module))
+
+    return True
 
 
-def load_macros(module_name):
+def load_macros(module):
     """Load the hy builtin macros for module `module_name`.
 
     Modules from `hy.core` can only use the macros from CORE_MACROS.
     Other modules get the macros from CORE_MACROS and EXTRA_MACROS.
-
     """
-    for module in CORE_MACROS:
+    builtin_macros = CORE_MACROS
-        importlib.import_module(module)
 
-    if module_name.startswith("hy.core"):
+    if not module.__name__.startswith("hy.core"):
-        return
+        builtin_macros += EXTRA_MACROS
 
-    for module in EXTRA_MACROS:
+    module_macros = module.__dict__.setdefault('__macros__', {})
-        importlib.import_module(module)
+    module_tags = module.__dict__.setdefault('__tags__', {})
+
+    for builtin_mod_name in builtin_macros:
+        builtin_mod = importlib.import_module(builtin_mod_name)
+
+        # Make sure we don't overwrite macros in the module.
+        if hasattr(builtin_mod, '__macros__'):
+            module_macros.update({k: v
+                                  for k, v in builtin_mod.__macros__.items()
+                                  if k not in module_macros})
+        if hasattr(builtin_mod, '__tags__'):
+            module_tags.update({k: v
+                                for k, v in builtin_mod.__tags__.items()
+                                if k not in module_tags})
 
 
 def make_empty_fn_copy(fn):
@@ -139,7 +249,7 @@ def make_empty_fn_copy(fn):
         # can continue running. Unfortunately, the error message that might get
         # raised later on while expanding a macro might not make sense at all.
 
-        formatted_args = hy.inspect.format_args(fn)
+        formatted_args = format_args(fn)
         fn_str = 'lambda {}: None'.format(
             formatted_args.lstrip('(').rstrip(')'))
         empty_fn = eval(fn_str)
@@ -152,14 +262,45 @@ def make_empty_fn_copy(fn):
     return empty_fn
 
 
-def macroexpand(tree, compiler, once=False):
+def macroexpand(tree, module, compiler=None, once=False):
-    """Expand the toplevel macros for the `tree`.
+    """Expand the toplevel macros for the given Hy AST tree.
 
-    Load the macros from the given `compiler.module_name`, then expand the
+    Load the macros from the given `module`, then expand the (top-level) macros
-    (top-level) macros in `tree` until we no longer can.
+    in `tree` until we no longer can.
 
+    `HyExpression` resulting from macro expansions are assigned the module in
+    which the macro function is defined (determined using `inspect.getmodule`).
+    If the resulting `HyExpression` is itself macro expanded, then the
+    namespace of the assigned module is checked first for a macro corresponding
+    to the expression's head/car symbol.  If the head/car symbol of such a
+    `HyExpression` is not found among the macros of its assigned module's
+    namespace, the outer-most namespace--e.g.  the one given by the `module`
+    parameter--is used as a fallback.
+
+    Parameters
+    ----------
+    tree: HyObject or list
+        Hy AST tree.
+
+    module: str or types.ModuleType
+        Module used to determine the local namespace for macros.
+
+    compiler: HyASTCompiler, optional
+        The compiler object passed to expanded macros.
+
+    once: boolean, optional
+        Only expand the first macro in `tree`.
+
+    Returns
+    ------
+    out: HyObject
+        Returns a mutated tree with macros expanded.
     """
-    load_macros(compiler.module_name)
+    if not inspect.ismodule(module):
+        module = importlib.import_module(module)
+
+    assert not compiler or compiler.module == module
+
     while True:
 
         if not isinstance(tree, HyExpression) or tree == []:
@@ -170,24 +311,34 @@ def macroexpand(tree, compiler, once=False):
             break
 
         fn = mangle(fn)
-        m = _hy_macros[compiler.module_name].get(fn) or _hy_macros[None].get(fn)
+        expr_modules = (([] if not hasattr(tree, 'module') else [tree.module])
+            + [module])
+
+        # Choose the first namespace with the macro.
+        m = next((mod.__macros__[fn]
+                  for mod in expr_modules
+                  if fn in mod.__macros__),
+                 None)
         if not m:
             break
 
         opts = {}
         if m._hy_macro_pass_compiler:
+            if compiler is None:
+                from hy.compiler import HyASTCompiler
+                compiler = HyASTCompiler(module)
             opts['compiler'] = compiler
 
         try:
             m_copy = make_empty_fn_copy(m)
-            m_copy(compiler.module_name, *tree[1:], **opts)
+            m_copy(module.__name__, *tree[1:], **opts)
         except TypeError as e:
             msg = "expanding `" + str(tree[0]) + "': "
             msg += str(e).replace("<lambda>()", "", 1).strip()
             raise HyMacroExpansionError(tree, msg)
 
         try:
-            obj = m(compiler.module_name, *tree[1:], **opts)
+            obj = m(module.__name__, *tree[1:], **opts)
         except HyTypeError as e:
             if e.expression is None:
                 e.expression = tree
@@ -195,6 +346,10 @@ def macroexpand(tree, compiler, once=False):
         except Exception as e:
             msg = "expanding `" + str(tree[0]) + "': " + repr(e)
             raise HyMacroExpansionError(tree, msg)
+
+        if isinstance(obj, HyExpression):
+            obj.module = inspect.getmodule(m)
+
         tree = replace_hy_obj(obj, tree)
 
         if once:
@@ -203,25 +358,33 @@ def macroexpand(tree, compiler, once=False):
     tree = wrap_value(tree)
     return tree
 
-def macroexpand_1(tree, compiler):
+
+def macroexpand_1(tree, module, compiler=None):
     """Expand the toplevel macro from `tree` once, in the context of
     `compiler`."""
-    return macroexpand(tree, compiler, once=True)
+    return macroexpand(tree, module, compiler, once=True)
 
 
-def tag_macroexpand(tag, tree, compiler):
+def tag_macroexpand(tag, tree, module):
-    """Expand the tag macro "tag" with argument `tree`."""
+    """Expand the tag macro `tag` with argument `tree`."""
-    load_macros(compiler.module_name)
+    if not inspect.ismodule(module):
+        module = importlib.import_module(module)
+
+    expr_modules = (([] if not hasattr(tree, 'module') else [tree.module])
+        + [module])
+
+    # Choose the first namespace with the macro.
+    tag_macro = next((mod.__tags__[tag]
+                      for mod in expr_modules
+                      if tag in mod.__tags__),
+                     None)
 
-    tag_macro = _hy_tag[compiler.module_name].get(tag)
     if tag_macro is None:
-        try:
+        raise HyTypeError(tag, "'{0}' is not a defined tag macro.".format(tag))
-            tag_macro = _hy_tag[None][tag]
-        except KeyError:
-            raise HyTypeError(
-                tag,
-                "`{0}' is not a defined tag macro.".format(tag)
-            )
 
     expr = tag_macro(tree)
+
+    if isinstance(expr, HyExpression):
+        expr.module = inspect.getmodule(tag_macro)
+
     return replace_hy_obj(expr, tree)

hy/models.py

@@ -32,11 +32,12 @@ class HyObject(object):
     Generic Hy Object model. This is helpful to inject things into all the
     Hy lexing Objects at once.
     """
+    __properties__ = ["module", "start_line", "end_line", "start_column",
+                      "end_column"]
 
     def replace(self, other, recursive=False):
         if isinstance(other, HyObject):
-            for attr in ["start_line", "end_line",
+            for attr in self.__properties__:
-                         "start_column", "end_column"]:
                 if not hasattr(self, attr) and hasattr(other, attr):
                     setattr(self, attr, getattr(other, attr))
         else:

tests/importer/test_importer.py

@@ -56,7 +56,7 @@ def test_runpy():
 
 
 def test_stringer():
-    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '')
+    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '__main__')
 
     assert type(_ast.body[0]) == ast.FunctionDef
 
@@ -80,7 +80,7 @@ def test_import_error_reporting():
 
     def _import_error_test():
         try:
-            _ = hy_compile(hy_parse("(import \"sys\")"), '')
+            _ = hy_compile(hy_parse("(import \"sys\")"), '__main__')
         except HyTypeError:
             return "Error reported"
 

tests/macros/test_macro_processor.py

@@ -22,6 +22,7 @@ def tmac(ETname, *tree):
 def test_preprocessor_simple():
     """ Test basic macro expansion """
     obj = macroexpand(tokenize('(test "one" "two")')[0],
+                      __name__,
                       HyASTCompiler(__name__))
     assert obj == HyList(["one", "two"])
     assert type(obj) == HyList
@@ -30,6 +31,7 @@ def test_preprocessor_simple():
 def test_preprocessor_expression():
     """ Test that macro expansion doesn't recurse"""
     obj = macroexpand(tokenize('(test (test "one" "two"))')[0],
+                      __name__,
                       HyASTCompiler(__name__))
 
     assert type(obj) == HyList
@@ -41,13 +43,13 @@ def test_preprocessor_expression():
 
     obj = HyList([HyString("one"), HyString("two")])
     obj = tokenize('(shill ["one" "two"])')[0][1]
-    assert obj == macroexpand(obj, HyASTCompiler(""))
+    assert obj == macroexpand(obj, __name__, HyASTCompiler(__name__))
 
 
 def test_preprocessor_exceptions():
     """ Test that macro expansion raises appropriate exceptions"""
     with pytest.raises(HyMacroExpansionError) as excinfo:
-        macroexpand(tokenize('(defn)')[0], HyASTCompiler(__name__))
+        macroexpand(tokenize('(defn)')[0], __name__, HyASTCompiler(__name__))
     assert "_hy_anon_fn_" not in excinfo.value.message
     assert "TypeError" not in excinfo.value.message
 
@@ -56,6 +58,6 @@ def test_macroexpand_nan():
    # https://github.com/hylang/hy/issues/1574
    import math
    NaN = float('nan')
-   x = macroexpand(HyFloat(NaN), HyASTCompiler(__name__))
+   x = macroexpand(HyFloat(NaN), __name__, HyASTCompiler(__name__))
    assert type(x) is HyFloat
    assert math.isnan(x)

tests/macros/test_tag_macros.py

@@ -11,6 +11,7 @@ def test_tag_macro_error():
     """Check if we get correct error with wrong dispatch character"""
     try:
         macroexpand(tokenize("(dispatch_tag_macro '- '())")[0],
+                    __name__,
                     HyASTCompiler(__name__))
     except HyTypeError as e:
         assert "with the character `-`" in str(e)

tests/native_tests/native_macros.hy

@@ -2,7 +2,8 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import [hy.errors [HyTypeError]])
+(import pytest
+        [hy.errors [HyTypeError]])
 
 (defmacro rev [&rest body]
   "Execute the `body` statements in reverse"
@@ -329,3 +330,124 @@
     (except [e SystemExit]
       (assert (= (str e) "42"))))
   (setv --name-- oldname))
+
+(defn test-macro-namespace-resolution []
+  "Confirm that local versions of macro-macro dependencies do not shadow the
+versions from the macro's own module, but do resolve unbound macro references
+in expansions."
+
+  ;; `nonlocal-test-macro` is a macro used within
+  ;; `tests.resources.macro-with-require.test-module-macro`.
+  ;; Here, we introduce an equivalently named version in local scope that, when
+  ;; used, will expand to a different output string.
+  (defmacro nonlocal-test-macro [x]
+    (print "this is the local version of `nonlocal-test-macro`!"))
+
+  ;; Was the above macro created properly?
+  (assert (in "nonlocal_test_macro" __macros__))
+
+  (setv nonlocal-test-macro (get __macros__ "nonlocal_test_macro"))
+
+  (require [tests.resources.macro-with-require [*]])
+
+  ;; Make sure our local version wasn't overwritten by a faulty `require` of the
+  ;; one in tests.resources.macro-with-require.
+  (assert (= nonlocal-test-macro (get __macros__ "nonlocal_test_macro")))
+
+  (setv module-name-var "tests.native_tests.native_macros.test-macro-namespace-resolution")
+  (assert (= (+ "This macro was created in tests.resources.macros, "
+                "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution "
+                "and passed the value 2.")
+             (test-module-macro 2)))
+  (assert (= (+ "This macro was created in tests.resources.macros, "
+                "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution "
+                "and passed the value 2.")
+             #test-module-tag 2))
+
+  ;; Now, let's use a `require`d macro that depends on another macro defined only
+  ;; in this scope.
+  (defmacro local-test-macro [x]
+    (.format "This is the local version of `nonlocal-test-macro` returning {}!" x))
+
+  (assert (= "This is the local version of `nonlocal-test-macro` returning 3!"
+             (test-module-macro-2 3)))
+  (assert (= "This is the local version of `nonlocal-test-macro` returning 3!"
+             #test-module-tag-2 3)))
+
+(defn test-macro-from-module []
+  "Macros loaded from an external module, which itself `require`s macros, should
+ work without having to `require` the module's macro dependencies (due to
+ [minimal] macro namespace resolution).
+
+ In doing so we also confirm that a module's `__macros__` attribute is correctly
+ loaded and used.
+
+ Additionally, we confirm that `require` statements are executed via loaded bytecode."
+
+  (import os sys marshal types)
+  (import [hy.importer [cache-from-source]])
+
+  (setv pyc-file (cache-from-source
+                   (os.path.realpath
+                     (os.path.join
+                       "tests" "resources" "macro_with_require.hy"))))
+
+  ;; Remove any cached byte-code, so that this runs from source and
+  ;; gets evaluated in this module.
+  (when (os.path.isfile pyc-file)
+    (os.unlink pyc-file)
+    (.clear sys.path_importer_cache)
+    (when (in  "tests.resources.macro_with_require" sys.modules)
+      (del (get sys.modules "tests.resources.macro_with_require"))
+      (__macros__.clear)
+      (__tags__.clear)))
+
+  ;; Ensure that bytecode isn't present when we require this module.
+  (assert (not (os.path.isfile pyc-file)))
+
+  (defn test-requires-and-macros []
+    (require [tests.resources.macro-with-require
+              [test-module-macro]])
+
+    ;; Make sure that `require` didn't add any of its `require`s
+    (assert (not (in "nonlocal-test-macro" __macros__)))
+    ;; and that it didn't add its tags.
+    (assert (not (in "test_module_tag" __tags__)))
+
+    ;; Now, require everything.
+    (require [tests.resources.macro-with-require [*]])
+
+    ;; Again, make sure it didn't add its required macros and/or tags.
+    (assert (not (in "nonlocal-test-macro" __macros__)))
+
+    ;; Its tag(s) should be here now.
+    (assert (in "test_module_tag" __tags__))
+
+    ;; The test macro expands to include this symbol.
+    (setv module-name-var "tests.native_tests.native_macros")
+    (assert (= (+ "This macro was created in tests.resources.macros, "
+                  "expanded in tests.native_tests.native_macros "
+                  "and passed the value 1.")
+               (test-module-macro 1)))
+
+    (assert (= (+ "This macro was created in tests.resources.macros, "
+                  "expanded in tests.native_tests.native_macros "
+                  "and passed the value 1.")
+               #test-module-tag 1)))
+
+  (test-requires-and-macros)
+
+  ;; Now that bytecode is present, reload the module, clear the `require`d
+  ;; macros and tags, and rerun the tests.
+  (assert (os.path.isfile pyc-file))
+
+  ;; Reload the module and clear the local macro context.
+  (.clear sys.path_importer_cache)
+  (del (get sys.modules "tests.resources.macro_with_require"))
+  (.clear __macros__)
+  (.clear __tags__)
+
+  ;; XXX: There doesn't seem to be a way--via standard import mechanisms--to
+  ;; ensure that an imported module used the cached bytecode.  We'll simply have
+  ;; to trust that the .pyc loading convention was followed.
+  (test-requires-and-macros))

tests/resources/bin/circular_macro_require.hy

@@ -0,0 +1,8 @@
+(defmacro bar [expr]
+  `(print ~expr))
+
+(defmacro foo [expr]
+  `(do (require [tests.resources.bin.circular-macro-require [bar]])
+       (bar ~expr)))
+
+(foo 42)

tests/resources/bin/require_and_eval.hy

@@ -0,0 +1,3 @@
+(require [hy.extra.anaphoric [ap-if]])
+
+(print (eval '(ap-if (+ "a" "b") (+ it "c"))))

tests/resources/macro_with_require.hy

@@ -0,0 +1,25 @@
+;; Require all the macros and make sure they don't pollute namespaces/modules
+;; that require `*` from this.
+(require [tests.resources.macros [*]])
+
+(defmacro test-module-macro [a]
+  "The variable `macro-level-var' here should not bind to the same-named symbol
+in the expansion of `nonlocal-test-macro'."
+  (setv macro-level-var "tests.resources.macros.macro-with-require")
+  `(nonlocal-test-macro ~a))
+
+(deftag test-module-tag [a]
+  "The variable `macro-level-var' here should not bind to the same-named symbol
+in the expansion of `nonlocal-test-macro'."
+  (setv macro-level-var "tests.resources.macros.macro-with-require")
+  `(nonlocal-test-macro ~a))
+
+(defmacro test-module-macro-2 [a]
+  "The macro `local-test-macro` isn't in this module's namespace, so it better
+ be in the expansion's!"
+  `(local-test-macro ~a))
+
+(deftag test-module-tag-2 [a]
+  "The macro `local-test-macro` isn't in this module's namespace, so it better
+ be in the expansion's!"
+  `(local-test-macro ~a))

tests/resources/macros.hy

@@ -1,3 +1,5 @@
+(setv module-name-var "tests.resources.macros")
+
 (defmacro thread-set-ab []
   (defn f [&rest args] (.join "" (+ (, "a") args)))
   (setv variable (HySymbol (-> "b" (f))))
@@ -10,3 +12,11 @@
 
 (defmacro test-macro []
   '(setv blah 1))
+
+(defmacro nonlocal-test-macro [x]
+  "When called from `macro-with-require`'s macro(s), the first instance of
+`module-name-var` should resolve to the value in the module where this is
+defined, then the expansion namespace/module"
+  `(.format (+ "This macro was created in {}, expanded in {} "
+               "and passed the value {}.")
+            ~module-name-var module-name-var ~x))

tests/test_bin.py

@@ -388,3 +388,38 @@ def test_bin_hy_file_no_extension():
     """Confirm that a file with no extension is processed as Hy source"""
     output, _ = run_cmd("hy tests/resources/no_extension")
     assert "This Should Still Work" in output
+
+
+def test_bin_hy_circular_macro_require():
+    """Confirm that macros can require themselves during expansion and when
+    run from the command line."""
+
+    # First, with no bytecode
+    test_file = "tests/resources/bin/circular_macro_require.hy"
+    rm(cache_from_source(test_file))
+    assert not os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "42" == output.strip()
+
+    # Now, with bytecode
+    assert os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "42" == output.strip()
+
+def test_bin_hy_macro_require():
+    """Confirm that a `require` will load macros into the non-module namespace
+    (i.e. `exec(code, locals)`) used by `runpy.run_path`.
+    In other words, this confirms that the AST generated for a `require` will
+    load macros into the unnamed namespace its run in."""
+
+    # First, with no bytecode
+    test_file = "tests/resources/bin/require_and_eval.hy"
+    rm(cache_from_source(test_file))
+    assert not os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "abc" == output.strip()
+
+    # Now, with bytecode
+    assert os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "abc" == output.strip()