Make macros module-specific.

A macro is available in the module where it was defined and
in any module that does a require of the defining module.
Only macros defined in hy.core are globally available.
Fixes #181
This commit is contained in:
Konrad Hinsen 2013-05-16 15:34:14 +02:00
parent b7c5ff2991
commit 269da19d76
11 changed files with 95 additions and 46 deletions

View File

@ -8,12 +8,13 @@ import astor.codegen
import sys import sys
import ast import ast
module_name = "<STDIN>"
hst = import_file_to_hst(sys.argv[1]) hst = import_file_to_hst(sys.argv[1])
print(hst) print(hst)
print("") print("")
print("") print("")
_ast = import_file_to_ast(sys.argv[1]) _ast = import_file_to_ast(sys.argv[1], module_name)
print("") print("")
print("") print("")
print(ast.dump(_ast)) print(ast.dump(_ast))
@ -21,4 +22,4 @@ print("")
print("") print("")
print(astor.codegen.to_source(_ast)) print(astor.codegen.to_source(_ast))
import_file_to_module("<STDIN>", sys.argv[1]) import_file_to_module(module_name, sys.argv[1])

View File

@ -41,7 +41,7 @@ from hy.importer import ast_compile, import_buffer_to_module
import hy.completer import hy.completer
from hy.macros import macro from hy.macros import macro, require
from hy.models.expression import HyExpression from hy.models.expression import HyExpression
from hy.models.string import HyString from hy.models.string import HyString
from hy.models.symbol import HySymbol from hy.models.symbol import HySymbol
@ -66,7 +66,7 @@ class HyREPL(code.InteractiveConsole):
return True return True
try: try:
tokens = process(_machine.nodes) tokens = process(_machine.nodes, "__console__")
except Exception: except Exception:
_machine = Machine(Idle, 1, 0) _machine = Machine(Idle, 1, 0)
self.showtraceback() self.showtraceback()
@ -74,7 +74,7 @@ class HyREPL(code.InteractiveConsole):
_machine = Machine(Idle, 1, 0) _machine = Machine(Idle, 1, 0)
try: try:
_ast = hy_compile(tokens, root=ast.Interactive) _ast = hy_compile(tokens, "__console__", root=ast.Interactive)
code = ast_compile(_ast, filename, symbol) code = ast_compile(_ast, filename, symbol)
except Exception: except Exception:
self.showtraceback() self.showtraceback()
@ -135,6 +135,9 @@ def ideas_macro():
""")]) """)])
require("hy.cmdline", "__console__")
require("hy.cmdline", "__main__")
def run_command(source): def run_command(source):
try: try:

View File

@ -40,6 +40,8 @@ from hy.core import process
from hy.util import str_type from hy.util import str_type
from hy.macros import require
import codecs import codecs
import traceback import traceback
import ast import ast
@ -321,10 +323,11 @@ def checkargs(exact=None, min=None, max=None):
class HyASTCompiler(object): class HyASTCompiler(object):
def __init__(self): def __init__(self, module_name):
self.anon_fn_count = 0 self.anon_fn_count = 0
self.anon_var_count = 0 self.anon_var_count = 0
self.imports = defaultdict(set) self.imports = defaultdict(set)
self.module_name = module_name
def get_anon_var(self): def get_anon_var(self):
self.anon_var_count += 1 self.anon_var_count += 1
@ -366,7 +369,7 @@ class HyASTCompiler(object):
def compile_atom(self, atom_type, atom): def compile_atom(self, atom_type, atom):
if atom_type in _compile_table: if atom_type in _compile_table:
atom = process(atom) atom = process(atom, self.module_name)
ret = _compile_table[atom_type](self, atom) ret = _compile_table[atom_type](self, atom)
if not isinstance(ret, Result): if not isinstance(ret, Result):
ret = Result() + ret ret = Result() + ret
@ -599,7 +602,8 @@ class HyASTCompiler(object):
ret = self.compile(HyExpression([ ret = self.compile(HyExpression([
HySymbol("hy_eval")] + expr + [ HySymbol("hy_eval")] + expr + [
HyExpression([HySymbol("locals")])]).replace(expr)) HyExpression([HySymbol("locals")])] + [
HyString(self.module_name)]).replace(expr))
ret.add_imports("hy.importer", ["hy_eval"]) ret.add_imports("hy.importer", ["hy_eval"])
@ -1225,6 +1229,7 @@ class HyASTCompiler(object):
expression.pop(0) expression.pop(0)
for entry in expression: for entry in expression:
__import__(entry) # Import it fo' them macros. __import__(entry) # Import it fo' them macros.
require(entry, self.module_name)
return Result() return Result()
@builds("and") @builds("and")
@ -1623,7 +1628,11 @@ class HyASTCompiler(object):
]).replace(expression) ]).replace(expression)
# Compile-time hack: we want to get our new macro now # Compile-time hack: we want to get our new macro now
hy.importer.hy_eval(new_expression, {'hy': hy}) # We must provide __name__ in the namespace to make the Python
# compiler set the __module__ attribute of the macro function.
hy.importer.hy_eval(new_expression,
{'hy': hy, '__name__': self.module_name},
self.module_name)
# We really want to have a `hy` import to get hy.macro in # We really want to have a `hy` import to get hy.macro in
ret = self.compile(new_expression) ret = self.compile(new_expression)
@ -1694,7 +1703,7 @@ class HyASTCompiler(object):
return ret return ret
def hy_compile(tree, root=ast.Module, get_expr=False): def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
""" """
Compile a HyObject tree into a Python AST Module. Compile a HyObject tree into a Python AST Module.
@ -1711,7 +1720,7 @@ def hy_compile(tree, root=ast.Module, get_expr=False):
expr = None expr = None
if tree: if tree:
compiler = HyASTCompiler() compiler = HyASTCompiler(module_name)
result = compiler.compile(tree) result = compiler.compile(tree)
expr = result.force_expr expr = result.force_expr

View File

@ -25,12 +25,12 @@ MACROS = [
] ]
def process(tree): def process(tree, module_name):
load_macros() load_macros()
old = None old = None
while old != tree: while old != tree:
old = tree old = tree
tree = mprocess(tree) tree = mprocess(tree, module_name)
return tree return tree

View File

@ -56,21 +56,21 @@ def import_file_to_hst(fpath):
return import_buffer_to_hst(f.read()) return import_buffer_to_hst(f.read())
def import_buffer_to_ast(buf): def import_buffer_to_ast(buf, module_name):
""" Import content from buf and return a Python AST.""" """ Import content from buf and return a Python AST."""
return hy_compile(import_buffer_to_hst(buf)) return hy_compile(import_buffer_to_hst(buf), module_name)
def import_file_to_ast(fpath): def import_file_to_ast(fpath, module_name):
"""Import content from fpath and return a Python AST.""" """Import content from fpath and return a Python AST."""
return hy_compile(import_file_to_hst(fpath)) return hy_compile(import_file_to_hst(fpath), module_name)
def import_file_to_module(module_name, fpath): def import_file_to_module(module_name, fpath):
"""Import content from fpath and puts it into a Python module. """Import content from fpath and puts it into a Python module.
Returns the module.""" Returns the module."""
_ast = import_file_to_ast(fpath) _ast = import_file_to_ast(fpath, module_name)
mod = imp.new_module(module_name) mod = imp.new_module(module_name)
mod.__file__ = fpath mod.__file__ = fpath
eval(ast_compile(_ast, fpath, "exec"), mod.__dict__) eval(ast_compile(_ast, fpath, "exec"), mod.__dict__)
@ -78,20 +78,20 @@ def import_file_to_module(module_name, fpath):
def import_buffer_to_module(module_name, buf): def import_buffer_to_module(module_name, buf):
_ast = import_buffer_to_ast(buf) _ast = import_buffer_to_ast(buf, module_name)
mod = imp.new_module(module_name) mod = imp.new_module(module_name)
eval(ast_compile(_ast, "", "exec"), mod.__dict__) eval(ast_compile(_ast, "", "exec"), mod.__dict__)
return mod return mod
def hy_eval(hytree, namespace): def hy_eval(hytree, namespace, module_name):
foo = HyObject() foo = HyObject()
foo.start_line = 0 foo.start_line = 0
foo.end_line = 0 foo.end_line = 0
foo.start_column = 0 foo.start_column = 0
foo.end_column = 0 foo.end_column = 0
hytree.replace(foo) hytree.replace(foo)
_ast, expr = hy_compile(hytree, get_expr=True) _ast, expr = hy_compile(hytree, module_name, get_expr=True)
# Spoof the positions in the generated ast... # Spoof the positions in the generated ast...
for node in ast.walk(_ast): for node in ast.walk(_ast):
@ -117,7 +117,8 @@ def write_hy_as_pyc(fname):
st = os.stat(fname) st = os.stat(fname)
timestamp = long_type(st.st_mtime) timestamp = long_type(st.st_mtime)
_ast = import_file_to_ast(fname) _ast = import_file_to_ast(fname,
os.path.basename(os.path.splitext(fname)[0]))
code = ast_compile(_ast, fname, "exec") code = ast_compile(_ast, fname, "exec")
cfile = "%s.pyc" % fname[:-len(".hy")] cfile = "%s.pyc" % fname[:-len(".hy")]

View File

@ -23,27 +23,41 @@ from hy.models.string import HyString
from hy.models.dict import HyDict from hy.models.dict import HyDict
from hy.models.list import HyList from hy.models.list import HyList
_hy_macros = {} from collections import defaultdict
_hy_macros = defaultdict(dict)
def macro(name): def macro(name):
def _(fn): def _(fn):
_hy_macros[name] = fn module_name = fn.__module__
if module_name.startswith("hy.core"):
module_name = None
_hy_macros[module_name][name] = fn
return fn return fn
return _ return _
def process(tree): def require(source_module_name, target_module_name):
macros = _hy_macros[source_module_name]
refs = _hy_macros[target_module_name]
for name, macro in macros.items():
refs[name] = macro
def process(tree, module_name):
if isinstance(tree, HyExpression): if isinstance(tree, HyExpression):
fn = tree[0] fn = tree[0]
if fn in ("quote", "quasiquote"): if fn in ("quote", "quasiquote"):
return tree return tree
ntree = HyExpression([fn] + [process(x) for x in tree[1:]]) ntree = HyExpression([fn] + [process(x, module_name) for x in tree[1:]])
ntree.replace(tree) ntree.replace(tree)
if isinstance(fn, HyString): if isinstance(fn, HyString):
if fn in _hy_macros: m = _hy_macros[module_name].get(fn)
m = _hy_macros[fn] if m is None:
m = _hy_macros[None].get(fn)
if m is not None:
obj = m(*ntree[1:]) obj = m(*ntree[1:])
obj.replace(tree) obj.replace(tree)
return obj return obj
@ -52,17 +66,19 @@ def process(tree):
return ntree return ntree
if isinstance(tree, HyDict): if isinstance(tree, HyDict):
obj = HyDict(dict((process(x), process(tree[x])) for x in tree)) obj = HyDict(dict((process(x, module_name),
process(tree[x], module_name))
for x in tree))
obj.replace(tree) obj.replace(tree)
return obj return obj
if isinstance(tree, HyList): if isinstance(tree, HyList):
obj = HyList([process(x) for x in tree]) # NOQA obj = HyList([process(x, module_name) for x in tree]) # NOQA
# flake8 thinks we're redefining from 52. # flake8 thinks we're redefining from 52.
obj.replace(tree) obj.replace(tree)
return obj return obj
if isinstance(tree, list): if isinstance(tree, list):
return [process(x) for x in tree] return [process(x, module_name) for x in tree]
return tree return tree

View File

@ -39,13 +39,13 @@ def _ast_spotcheck(arg, root, secondary):
def can_compile(expr): def can_compile(expr):
return hy_compile(tokenize(expr)) return hy_compile(tokenize(expr), "__main__")
def cant_compile(expr): def cant_compile(expr):
expr = tokenize(expr) expr = tokenize(expr)
try: try:
hy_compile(expr) hy_compile(expr, "__main__")
assert False assert False
except HyCompileError: except HyCompileError:
pass pass
@ -54,7 +54,7 @@ def cant_compile(expr):
def test_ast_bad_type(): def test_ast_bad_type():
"Make sure AST breakage can happen" "Make sure AST breakage can happen"
try: try:
hy_compile("foo") hy_compile("foo", "__main__")
assert True is False assert True is False
except HyCompileError: except HyCompileError:
pass pass
@ -350,7 +350,7 @@ def test_ast_non_kwapplyable():
code = tokenize("(kwapply foo bar)") code = tokenize("(kwapply foo bar)")
code[0][2] = None code[0][2] = None
try: try:
hy_compile(code) hy_compile(code, "__main__")
assert True is False assert True is False
except HyCompileError: except HyCompileError:
pass pass
@ -410,7 +410,7 @@ def test_ast_unicode_strings():
hy_s.start_line = hy_s.end_line = 0 hy_s.start_line = hy_s.end_line = 0
hy_s.start_column = hy_s.end_column = 0 hy_s.start_column = hy_s.end_column = 0
code = hy_compile([hy_s]) code = hy_compile([hy_s], "__main__")
# code == ast.Module(body=[ast.Expr(value=ast.Str(s=xxx))]) # code == ast.Module(body=[ast.Expr(value=ast.Str(s=xxx))])
return code.body[0].value.s return code.body[0].value.s

View File

@ -46,7 +46,7 @@ class HyASTCompilerTest(unittest.TestCase):
return h return h
def setUp(self): def setUp(self):
self.c = HyASTCompiler() self.c = HyASTCompiler('')
def test_fn_compiler_empty_function(self): def test_fn_compiler_empty_function(self):
ret = self.c.compile_function_def( ret = self.c.compile_function_def(

View File

@ -10,5 +10,5 @@ def test_basics():
def test_stringer(): def test_stringer():
"Make sure the basics of the importer work" "Make sure the basics of the importer work"
_ast = import_buffer_to_ast("(defn square [x] (* x x))") _ast = import_buffer_to_ast("(defn square [x] (* x x))", '')
assert type(_ast.body[0]) == ast.FunctionDef assert type(_ast.body[0]) == ast.FunctionDef

View File

@ -13,15 +13,15 @@ def tmac(*tree):
def test_preprocessor_simple(): def test_preprocessor_simple():
""" Test basic macro expantion """ """ Test basic macro expansion """
obj = process(tokenize('(test "one" "two")')[0]) obj = process(tokenize('(test "one" "two")')[0], __name__)
assert obj == HyList(["one", "two"]) assert obj == HyList(["one", "two"])
assert type(obj) == HyList assert type(obj) == HyList
def test_preprocessor_expression(): def test_preprocessor_expression():
""" Test inner macro expantion """ """ Test inner macro expansion """
obj = process(tokenize('(test (test "one" "two"))')[0]) obj = process(tokenize('(test (test "one" "two"))')[0], __name__)
assert type(obj) == HyList assert type(obj) == HyList
assert type(obj[0]) == HyList assert type(obj[0]) == HyList
@ -30,4 +30,4 @@ def test_preprocessor_expression():
obj = HyList([HyString("one"), HyString("two")]) obj = HyList([HyString("one"), HyString("two")])
obj = tokenize('(shill ["one" "two"])')[0][1] obj = tokenize('(shill ["one" "two"])')[0][1]
assert obj == process(obj) assert obj == process(obj, '')

View File

@ -699,7 +699,26 @@
(defn test-require-native [] (defn test-require-native []
"NATIVE: test requiring macros from native code" "NATIVE: test requiring macros from native code"
(assert (= "failure"
(try
(do (setv x [])
(rev (.append x 1) (.append x 2) (.append x 3))
(assert (= x [3 2 1]))
"success")
(except [NameError] "failure"))))
(import tests.native_tests.native_macros)
(assert (= "failure"
(try
(do (setv x [])
(rev (.append x 1) (.append x 2) (.append x 3))
(assert (= x [3 2 1]))
"success")
(except [NameError] "failure"))))
(require tests.native_tests.native_macros) (require tests.native_tests.native_macros)
(setv x []) (assert (= "success"
(rev (.append x 1) (.append x 2) (.append x 3)) (try
(assert (= x [3 2 1]))) (do (setv x [])
(rev (.append x 1) (.append x 2) (.append x 3))
(assert (= x [3 2 1]))
"success")
(except [NameError] "failure")))))