Overhaul macros to allow macros to ref the Compiler

This allows macros to take a keyword dict containing useful things by
defining a keyword argument. This allows us to pass in new objects
which might be handy to have in macros.

This changeset refactors module_name to become `compiler`, so that we
can pass the compiler itself into the macros as `opts['compiler']`.

This allows the macro to both get the macro name
(`compiler.module_name`), as well as use the compiler to build AST.

In the future, this will enable us to create "super-macros" which return
AST, not HyAST, in order to manually create insane things from userland.

For userland macros (not `defmacro`) the core.language `macroexpand`
will go ahead and make a new compiler for you.

hy/compiler.py

@@ -1989,7 +1989,7 @@ class HyASTCompiler(object):
     @builds(HyExpression)
     def compile_expression(self, expression):
         # Perform macro expansions
-        expression = macroexpand(expression, self.module_name)
+        expression = macroexpand(expression, self)
         if not isinstance(expression, HyExpression):
             # Go through compile again if the type changed.
             return self.compile(expression)
@@ -2396,7 +2396,7 @@ class HyASTCompiler(object):
             body += self.compile(rewire_init(expr))
 
         for expression in expressions:
-            expr = rewire_init(macroexpand(expression, self.module_name))
+            expr = rewire_init(macroexpand(expression, self))
             body += self.compile(expr)
 
         self.allow_builtins = allow_builtins
@@ -2482,10 +2482,7 @@ class HyASTCompiler(object):
                 "Trying to expand a reader macro using `{0}' instead "
                 "of string".format(type(str_char).__name__),
             )
-
+        expr = reader_macroexpand(str_char, expression.pop(0), self)
-        module = self.module_name
-        expr = reader_macroexpand(str_char, expression.pop(0), module)
-
         return self.compile(expr)
 
     @builds("eval_and_compile")

hy/core/language.hy

@@ -37,6 +37,7 @@
         [hy.models.symbol [HySymbol]]
         [hy.models.keyword [HyKeyword *keyword-prefix*]])
 (import [hy.lex [LexException PrematureEndOfInput tokenize]])
+(import [hy.compiler [HyASTCompiler]])
 
 (defn _numeric-check [x]
   (if (not (numeric? x))
@@ -296,14 +297,14 @@
   (import hy.macros)
 
   (setv name (calling-module-name))
-  (hy.macros.macroexpand form name))
+  (hy.macros.macroexpand form (HyASTCompiler name)))
 
 (defn macroexpand-1 [form]
   "Return the single step macro expansion of form"
   (import hy.macros)
 
   (setv name (calling-module-name))
-  (hy.macros.macroexpand-1 form name))
+  (hy.macros.macroexpand-1 form (HyASTCompiler name)))
 
 (defn merge-with [f &rest maps]
   "Returns a map that consists of the rest of the maps joined onto

hy/macros.py

@@ -52,6 +52,8 @@ def macro(name):
 
     """
     def _(fn):
+        argspec = getargspec(fn)
+        fn._hy_macro_pass_compiler = argspec.keywords is not None
         module_name = fn.__module__
         if module_name.startswith("hy.core"):
             module_name = None
@@ -133,22 +135,22 @@ def make_empty_fn_copy(fn):
     return empty_fn
 
 
-def macroexpand(tree, module_name):
+def macroexpand(tree, compiler):
     """Expand the toplevel macros for the `tree`.
 
     Load the macros from the given `module_name`, then expand the (top-level)
     macros in `tree` until it stops changing.
 
     """
-    load_macros(module_name)
+    load_macros(compiler.module_name)
     old = None
     while old != tree:
         old = tree
-        tree = macroexpand_1(tree, module_name)
+        tree = macroexpand_1(tree, compiler)
     return tree
 
 
-def macroexpand_1(tree, module_name):
+def macroexpand_1(tree, compiler):
     """Expand the toplevel macro from `tree` once, in the context of
     `module_name`."""
     if isinstance(tree, HyExpression):
@@ -161,22 +163,25 @@ def macroexpand_1(tree, module_name):
         ntree = HyExpression(tree[:])
         ntree.replace(tree)
 
+        opts = {}
+
         if isinstance(fn, HyString):
-            m = _hy_macros[module_name].get(fn)
+            m = _hy_macros[compiler.module_name].get(fn)
             if m is None:
                 m = _hy_macros[None].get(fn)
             if m is not None:
+                if m._hy_macro_pass_compiler:
+                    opts['compiler'] = compiler
+
                 try:
                     m_copy = make_empty_fn_copy(m)
-                    m_copy(*ntree[1:])
+                    m_copy(*ntree[1:], **opts)
                 except TypeError as e:
                     msg = "expanding `" + str(tree[0]) + "': "
                     msg += str(e).replace("<lambda>()", "", 1).strip()
                     raise HyMacroExpansionError(tree, msg)
-
                 try:
-                    obj = wrap_value(m(*ntree[1:]))
+                    obj = wrap_value(m(*ntree[1:], **opts))
-
                 except HyTypeError as e:
                     if e.expression is None:
                         e.expression = tree
@@ -186,16 +191,15 @@ def macroexpand_1(tree, module_name):
                     raise HyMacroExpansionError(tree, msg)
                 replace_hy_obj(obj, tree)
                 return obj
-
         return ntree
     return tree
 
 
-def reader_macroexpand(char, tree, module_name):
+def reader_macroexpand(char, tree, compiler):
     """Expand the reader macro "char" with argument `tree`."""
-    load_macros(module_name)
+    load_macros(compiler.module_name)
 
-    reader_macro = _hy_reader[module_name].get(char)
+    reader_macro = _hy_reader[compiler.module_name].get(char)
     if reader_macro is None:
         try:
             reader_macro = _hy_reader[None][char]

tests/macros/test_macro_processor.py

@@ -8,6 +8,8 @@ from hy.models.symbol import HySymbol
 from hy.models.expression import HyExpression
 from hy.errors import HyMacroExpansionError
 
+from hy.compiler import HyASTCompiler
+
 
 @macro("test")
 def tmac(*tree):
@@ -17,14 +19,16 @@ def tmac(*tree):
 
 def test_preprocessor_simple():
     """ Test basic macro expansion """
-    obj = macroexpand(tokenize('(test "one" "two")')[0], __name__)
+    obj = macroexpand(tokenize('(test "one" "two")')[0],
+                      HyASTCompiler(__name__))
     assert obj == HyList(["one", "two"])
     assert type(obj) == HyList
 
 
 def test_preprocessor_expression():
     """ Test that macro expansion doesn't recurse"""
-    obj = macroexpand(tokenize('(test (test "one" "two"))')[0], __name__)
+    obj = macroexpand(tokenize('(test (test "one" "two"))')[0],
+                      HyASTCompiler(__name__))
 
     assert type(obj) == HyList
     assert type(obj[0]) == HyExpression
@@ -35,13 +39,13 @@ def test_preprocessor_expression():
 
     obj = HyList([HyString("one"), HyString("two")])
     obj = tokenize('(shill ["one" "two"])')[0][1]
-    assert obj == macroexpand(obj, '')
+    assert obj == macroexpand(obj, HyASTCompiler(""))
 
 
 def test_preprocessor_exceptions():
     """ Test that macro expansion raises appropriate exceptions"""
     try:
-        macroexpand(tokenize('(defn)')[0], __name__)
+        macroexpand(tokenize('(defn)')[0], HyASTCompiler(__name__))
         assert False
     except HyMacroExpansionError as e:
         assert "_hy_anon_fn_" not in str(e)

tests/macros/test_reader_macros.py

@@ -1,11 +1,12 @@
 from hy.macros import macroexpand
-from hy.compiler import HyTypeError
+from hy.compiler import HyTypeError, HyASTCompiler
 from hy.lex import tokenize
 
 
 def test_reader_macro_error():
     """Check if we get correct error with wrong dispatch character"""
     try:
-        macroexpand(tokenize("(dispatch_reader_macro '- '())")[0], __name__)
+        macroexpand(tokenize("(dispatch_reader_macro '- '())")[0],
+                    HyASTCompiler(__name__))
     except HyTypeError as e:
         assert "with the character `-`" in str(e)