diff --git a/NEWS.rst b/NEWS.rst index 5a67e80..bb101a2 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -12,9 +12,15 @@ New Features * Keyword objects (not just literal keywords) can be called, as shorthand for `(get obj :key)`, and they accept a default value as a second argument. +* Minimal macro expansion namespacing has been implemented. As a result, + external macros no longer have to `require` their own macro dependencies. +* Macros and tags now reside in module-level `__macros__` and `__tags__` + attributes. Bug Fixes ------------------------------ +* `require` now compiles to Python AST. +* Fixed circular `require`s. * Fixed module reloading. * Fixed circular imports. * Fixed `__main__` file execution. diff --git a/hy/cmdline.py b/hy/cmdline.py index f38355b..eeb5ccd 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -13,6 +13,7 @@ import io import importlib import py_compile import runpy +import types import astor.code_gen @@ -47,10 +48,26 @@ builtins.quit = HyQuitter('quit') builtins.exit = HyQuitter('exit') -class HyREPL(code.InteractiveConsole): +class HyREPL(code.InteractiveConsole, object): def __init__(self, spy=False, output_fn=None, locals=None, filename=""): + super(HyREPL, self).__init__(locals=locals, + filename=filename) + + # Create a proper module for this REPL so that we can obtain it easily + # (e.g. using `importlib.import_module`). + # Also, make sure it's properly introduced to `sys.modules` and + # consistently use its namespace as `locals` from here on. + module_name = self.locals.get('__name__', '__console__') + self.module = sys.modules.setdefault(module_name, + types.ModuleType(module_name)) + self.module.__dict__.update(self.locals) + self.locals = self.module.__dict__ + + # Load cmdline-specific macros. + require('hy.cmdline', module_name, assignments='ALL') + self.spy = spy if output_fn is None: @@ -65,9 +82,6 @@ class HyREPL(code.InteractiveConsole): else: self.output_fn = __builtins__[mangle(output_fn)] - code.InteractiveConsole.__init__(self, locals=locals, - filename=filename) - # Pre-mangle symbols for repl recent results: *1, *2, *3 self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] self.locals.update({sym: None for sym in self._repl_results_symbols}) @@ -102,8 +116,7 @@ class HyREPL(code.InteractiveConsole): new_ast = ast.Module(main_ast.body + [ast.Expr(expr_ast.body)]) print(astor.to_source(new_ast)) - value = hy_eval(do, self.locals, "__console__", - ast_callback) + value = hy_eval(do, self.locals, self.module, ast_callback) except HyTypeError as e: if e.source is None: e.source = source @@ -181,8 +194,6 @@ def ideas_macro(ETname): """)]) -require("hy.cmdline", "__console__", assignments="ALL") -require("hy.cmdline", "__main__", assignments="ALL") SIMPLE_TRACEBACKS = True @@ -199,7 +210,8 @@ def pretty_error(func, *args, **kw): def run_command(source): tree = hy_parse(source) - pretty_error(hy_eval, tree, module_name="__main__") + require("hy.cmdline", "__main__", assignments="ALL") + pretty_error(hy_eval, tree, None, importlib.import_module('__main__')) return 0 @@ -208,13 +220,13 @@ def run_repl(hr=None, **kwargs): sys.ps1 = "=> " sys.ps2 = "... " - namespace = {'__name__': '__console__', '__doc__': ''} + if not hr: + hr = HyREPL(**kwargs) + + namespace = hr.locals with completion(Completer(namespace)): - if not hr: - hr = HyREPL(locals=namespace, **kwargs) - hr.interact("{appname} {version} using " "{py}({build}) {pyversion} on {os}".format( appname=hy.__appname__, @@ -409,7 +421,6 @@ def hyc_main(): # entry point for cmd line script "hy2py" def hy2py_main(): import platform - module_name = "" options = dict(prog="hy2py", usage="%(prog)s [options] [FILE]", formatter_class=argparse.RawDescriptionHelpFormatter) @@ -448,7 +459,7 @@ def hy2py_main(): print() print() - _ast = pretty_error(hy_compile, hst, module_name) + _ast = pretty_error(hy_compile, hst, '__main__') if options.with_ast: if PY3 and platform.system() == "Windows": _print_for_windows(astor.dump_tree(_ast)) diff --git a/hy/compiler.py b/hy/compiler.py index bd43cf7..f6cf052 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -13,14 +13,16 @@ from hy.errors import HyCompileError, HyTypeError from hy.lex import mangle, unmangle -import hy.macros -from hy._compat import ( - str_type, bytes_type, long_type, PY3, PY35, raise_empty) -from hy.macros import require, macroexpand, tag_macroexpand +from hy._compat import (str_type, string_types, bytes_type, long_type, PY3, + PY35, raise_empty) +from hy.macros import require, load_macros, macroexpand, tag_macroexpand import hy.importer import traceback import importlib +import inspect +import pkgutil +import types import ast import sys import copy @@ -279,32 +281,48 @@ def is_unpack(kind, x): and x[0] == "unpack-" + kind) -_stdlib = {} - - class HyASTCompiler(object): + """A Hy-to-Python AST compiler""" - def __init__(self, module_name): + def __init__(self, module): + """ + Parameters + ---------- + module: str or types.ModuleType + Module in which the Hy tree is evaluated. + """ self.anon_var_count = 0 self.imports = defaultdict(set) - self.module_name = module_name self.temp_if = None + + if not inspect.ismodule(module): + module = importlib.import_module(module) + + self.module = module + self.module_name = module.__name__ + self.can_use_stdlib = ( - not module_name.startswith("hy.core") - or module_name == "hy.core.macros") + not self.module_name.startswith("hy.core") + or self.module_name == "hy.core.macros") + + # Load stdlib macros into the module namespace. + load_macros(self.module) + + self._stdlib = {} + # Everything in core needs to be explicit (except for # the core macros, which are built with the core functions). - if self.can_use_stdlib and not _stdlib: + if self.can_use_stdlib: # Populate _stdlib. import hy.core - for module in hy.core.STDLIB: - mod = importlib.import_module(module) - for e in map(ast_str, mod.EXPORTS): + for stdlib_module in hy.core.STDLIB: + mod = importlib.import_module(stdlib_module) + for e in map(ast_str, getattr(mod, 'EXPORTS', [])): if getattr(mod, e) is not getattr(builtins, e, ''): # Don't bother putting a name in _stdlib if it # points to a builtin with the same name. This # prevents pointless imports. - _stdlib[e] = module + self._stdlib[e] = stdlib_module def get_anon_var(self): self.anon_var_count += 1 @@ -1098,11 +1116,6 @@ class HyASTCompiler(object): brackets(SYM, sym(":as"), _symn) | brackets(SYM, brackets(many(_symn + maybe(sym(":as") + _symn)))))]) def compile_import_or_require(self, expr, root, entries): - """ - TODO for `require`: keep track of what we've imported in this run and - then "unimport" it after we've completed `thing' so that we don't - pollute other envs. - """ ret = Result() for entry in entries: @@ -1128,8 +1141,9 @@ class HyASTCompiler(object): else: assignments = [(k, v or k) for k, v in kids] + ast_module = ast_str(module, piecewise=True) + if root == "import": - ast_module = ast_str(module, piecewise=True) module = ast_module.lstrip(".") level = len(ast_module) - len(module) if assignments == "ALL" and prefix == "": @@ -1150,10 +1164,23 @@ class HyASTCompiler(object): for k, v in assignments] ret += node( expr, module=module or None, names=names, level=level) - else: # root == "require" - importlib.import_module(module) - require(module, self.module_name, - assignments=assignments, prefix=prefix) + + elif require(ast_module, self.module, assignments=assignments, + prefix=prefix): + # Actually calling `require` is necessary for macro expansions + # occurring during compilation. + self.imports['hy.macros'].update([None]) + # The `require` we're creating in AST is the same as above, but used at + # run-time (e.g. when modules are loaded via bytecode). + ret += self.compile(HyExpression([ + HySymbol('hy.macros.require'), + HyString(ast_module), + HySymbol('None'), + HyKeyword('assignments'), + (HyString("ALL") if assignments == "ALL" else + [[HyString(k), HyString(v)] for k, v in assignments]), + HyKeyword('prefix'), + HyString(prefix)]).replace(expr)) return ret @@ -1484,7 +1511,8 @@ class HyASTCompiler(object): [x for pair in attrs[0] for x in pair]).replace(attrs))) for e in body: - e = self.compile(self._rewire_init(macroexpand(e, self))) + e = self.compile(self._rewire_init( + macroexpand(e, self.module, self))) bodyr += e + e.expr_as_stmt() return bases + asty.ClassDef( @@ -1520,20 +1548,16 @@ class HyASTCompiler(object): return self.compile(tag_macroexpand( HyString(mangle(tag)).replace(tag), arg, - self)) - - _namespaces = {} + self.module)) @special(["eval-and-compile", "eval-when-compile"], [many(FORM)]) def compile_eval_and_compile(self, expr, root, body): new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr) - if self.module_name not in self._namespaces: - # Initialize a compile-time namespace for this module. - self._namespaces[self.module_name] = { - 'hy': hy, '__name__': self.module_name} + hy.importer.hy_eval(new_expr + body, - self._namespaces[self.module_name], - self.module_name) + self.module.__dict__, + self.module) + return (self._compile_branch(body) if ast_str(root) == "eval_and_compile" else Result()) @@ -1541,7 +1565,7 @@ class HyASTCompiler(object): @builds_model(HyExpression) def compile_expression(self, expr): # Perform macro expansions - expr = macroexpand(expr, self) + expr = macroexpand(expr, self.module, self) if not isinstance(expr, HyExpression): # Go through compile again if the type changed. return self.compile(expr) @@ -1665,8 +1689,8 @@ class HyASTCompiler(object): attr=ast_str(local), ctx=ast.Load()) - if self.can_use_stdlib and ast_str(symbol) in _stdlib: - self.imports[_stdlib[ast_str(symbol)]].add(ast_str(symbol)) + if self.can_use_stdlib and ast_str(symbol) in self._stdlib: + self.imports[self._stdlib[ast_str(symbol)]].add(ast_str(symbol)) return asty.Name(symbol, id=ast_str(symbol), ctx=ast.Load()) @@ -1699,20 +1723,41 @@ class HyASTCompiler(object): return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2]) -def hy_compile(tree, module_name, root=ast.Module, get_expr=False): +def hy_compile(tree, module, root=ast.Module, get_expr=False): """ - Compile a HyObject tree into a Python AST Module. + Compile a Hy tree into a Python AST tree. - If `get_expr` is True, return a tuple (module, last_expression), where - `last_expression` is the. + Parameters + ---------- + module: str or types.ModuleType + Module, or name of the module, in which the Hy tree is evaluated. + + root: ast object, optional (ast.Module) + Root object for the Python AST tree. + + get_expr: bool, optional (False) + If true, return a tuple with `(root_obj, last_expression)`. + + Returns + ------- + out : A Python AST tree """ + if isinstance(module, string_types): + if module.startswith('<') and module.endswith('>'): + module = types.ModuleType(module) + else: + module = importlib.import_module(ast_str(module, piecewise=True)) + if not inspect.ismodule(module): + raise TypeError('Invalid module type: {}'.format(type(module))) + + tree = wrap_value(tree) if not isinstance(tree, HyObject): raise HyCompileError("`tree` must be a HyObject or capable of " "being promoted to one") - compiler = HyASTCompiler(module_name) + compiler = HyASTCompiler(module) result = compiler.compile(tree) expr = result.force_expr diff --git a/hy/completer.py b/hy/completer.py index 7748c3d..9b7bb4f 100644 --- a/hy/completer.py +++ b/hy/completer.py @@ -39,13 +39,15 @@ class Completer(object): self.namespace = namespace self.path = [hy.compiler._special_form_compilers, builtins.__dict__, - hy.macros._hy_macros[None], namespace] - self.tag_path = [hy.macros._hy_tag[None]] - if '__name__' in namespace: - module_name = namespace['__name__'] - self.path.append(hy.macros._hy_macros[module_name]) - self.tag_path.append(hy.macros._hy_tag[module_name]) + + self.tag_path = [] + + namespace.setdefault('__macros__', {}) + namespace.setdefault('__tags__', {}) + + self.path.append(namespace['__macros__']) + self.tag_path.append(namespace['__tags__']) def attr_matches(self, text): # Borrowed from IPython's completer diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy index 4b6df7a..70fbca5 100644 --- a/hy/contrib/walk.hy +++ b/hy/contrib/walk.hy @@ -5,6 +5,7 @@ (import [hy [HyExpression HyDict]] [functools [partial]] + [importlib [import-module]] [collections [OrderedDict]] [hy.macros [macroexpand :as mexpand]] [hy.compiler [HyASTCompiler]]) @@ -42,9 +43,11 @@ (defn macroexpand-all [form &optional module-name] "Recursively performs all possible macroexpansions in form." - (setv module-name (or module-name (calling-module-name)) + (setv module (or (and module-name + (import-module module-name)) + (calling-module)) quote-level [0] - ast-compiler (HyASTCompiler module-name)) ; TODO: make nonlocal after dropping Python2 + ast-compiler (HyASTCompiler module)) ; TODO: make nonlocal after dropping Python2 (defn traverse [form] (walk expand identity form)) (defn expand [form] @@ -68,7 +71,7 @@ [(= (first form) (HySymbol "require")) (ast-compiler.compile form) (return)] - [True (traverse (mexpand form ast-compiler))]) + [True (traverse (mexpand form module ast-compiler))]) (if (coll? form) (traverse form) form))) diff --git a/hy/core/language.hy b/hy/core/language.hy index abb387f..5ada235 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -21,7 +21,7 @@ (import [hy.models [HySymbol HyKeyword]]) (import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]]) (import [hy.compiler [HyASTCompiler]]) -(import [hy.importer [hy-eval :as eval]]) +(import [hy.importer [calling-module hy-eval :as eval]]) (defn butlast [coll] "Return an iterator of all but the last item in `coll`." @@ -295,12 +295,14 @@ Return series of accumulated sums (or other binary function results)." (defn macroexpand [form] "Return the full macro expansion of `form`." (import hy.macros) - (hy.macros.macroexpand form (HyASTCompiler (calling-module-name)))) + (setv module (calling-module)) + (hy.macros.macroexpand form module (HyASTCompiler module))) (defn macroexpand-1 [form] "Return the single step macro expansion of `form`." (import hy.macros) - (hy.macros.macroexpand-1 form (HyASTCompiler (calling-module-name)))) + (setv module (calling-module)) + (hy.macros.macroexpand-1 form module (HyASTCompiler module))) (defn merge-with [f &rest maps] "Return the map of `maps` joined onto the first via the function `f`. @@ -467,8 +469,8 @@ Even objects with the __name__ magic will work." (or a b))) (setv EXPORTS - '[*map accumulate butlast calling-module-name chain coll? combinations - comp complement compress constantly count cycle dec distinct + '[*map accumulate butlast calling-module calling-module-name chain coll? + combinations comp complement compress constantly count cycle dec distinct disassemble drop drop-last drop-while empty? eval even? every? exec first filter flatten float? fraction gensym group-by identity inc input instance? integer integer? integer-char? interleave interpose islice iterable? diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 88b5de9..2f9154e 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -245,34 +245,10 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol." Use ``#doc foo`` instead for help with tag macro ``#foo``. Use ``(help foo)`` instead for help with runtime objects." - `(try - (import [importlib [import-module]]) - (help (. (import-module "hy") - macros - _hy_macros - [__name__] - ['~symbol])) - (except [KeyError] - (help (. (import-module "hy") - macros - _hy_macros - [None] - ['~symbol]))))) + `(help (.get __macros__ '~symbol None))) (deftag doc [symbol] "tag macro documentation Gets help for a tag macro function available in this module." - `(try - (import [importlib [import-module]]) - (help (. (import-module "hy") - macros - _hy_tag - [__name__] - ['~symbol])) - (except [KeyError] - (help (. (import-module "hy") - macros - _hy_tag - [None] - ['~symbol]))))) + `(help (.get __tags__ '~symbol None))) diff --git a/hy/extra/reserved.hy b/hy/extra/reserved.hy index 90ffee5..3de34ea 100644 --- a/hy/extra/reserved.hy +++ b/hy/extra/reserved.hy @@ -16,7 +16,7 @@ (setv _cache (frozenset (map unmangle (+ hy.core.language.EXPORTS hy.core.shadow.EXPORTS - (list (.keys (get hy.macros._hy_macros None))) + (list (.keys hy.core.macros.__macros__)) keyword.kwlist (list (.keys hy.compiler._special_form_compilers)) (list hy.compiler._bad_roots))))))) diff --git a/hy/importer.py b/hy/importer.py index 18bab3f..fdcb0be 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -17,9 +17,10 @@ import importlib import __future__ from functools import partial +from contextlib import contextmanager from hy.errors import HyTypeError -from hy.compiler import hy_compile +from hy.compiler import hy_compile, ast_str from hy.lex import tokenize, LexException from hy.models import HyExpression, HySymbol from hy._compat import string_types, PY3 @@ -29,6 +30,36 @@ hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION | __future__.CO_FUTURE_PRINT_FUNCTION) +def calling_module(n=1): + """Get the module calling, if available. + + As a fallback, this will import a module using the calling frame's + globals value of `__name__`. + + Parameters + ---------- + n: int, optional + The number of levels up the stack from this function call. + The default is one level up. + + Returns + ------- + out: types.ModuleType + The module at stack level `n + 1` or `None`. + """ + frame_up = inspect.stack(0)[n + 1][0] + module = inspect.getmodule(frame_up) + if module is None: + # This works for modules like `__main__` + module_name = frame_up.f_globals.get('__name__', None) + if module_name: + try: + module = importlib.import_module(module_name) + except ImportError: + pass + return module + + def ast_compile(ast, filename, mode): """Compile AST. @@ -65,13 +96,9 @@ def hy_parse(source): return HyExpression([HySymbol("do")] + tokenize(source + "\n")) -def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): +def hy_eval(hytree, locals=None, module=None, ast_callback=None): """Evaluates a quoted expression and returns the value. - The optional second and third arguments specify the dictionary of globals - to use and the module name. The globals dictionary defaults to ``(local)`` - and the module name defaults to the name of the current module. - Examples -------- @@ -89,13 +116,15 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): hytree: a Hy expression tree Source code to parse. - namespace: dict, optional - Namespace in which to evaluate the Hy tree. Defaults to the calling - frame. + locals: dict, optional + Local environment in which to evaluate the Hy tree. Defaults to the + calling frame. - module_name: str, optional - Name of the module to which the Hy tree is assigned. Defaults to - the calling frame's module, if any, and '__eval__' otherwise. + module: str or types.ModuleType, optional + Module, or name of the module, to which the Hy tree is assigned and + the global values are taken. + Defaults to the calling frame's module, if any, and '__eval__' + otherwise. ast_callback: callable, optional A callback that is passed the Hy compiled tree and resulting @@ -105,19 +134,23 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): Returns ------- out : Result of evaluating the Hy compiled tree. - """ - if namespace is None: + if module is None: + module = calling_module() + + if isinstance(module, string_types): + module = importlib.import_module(ast_str(module, piecewise=True)) + elif not inspect.ismodule(module): + raise TypeError('Invalid module type: {}'.format(type(module))) + + if locals is None: frame = inspect.stack()[1][0] - namespace = inspect.getargvalues(frame).locals - if module_name is None: - m = inspect.getmodule(inspect.stack()[1][0]) - module_name = '__eval__' if m is None else m.__name__ + locals = inspect.getargvalues(frame).locals - if not isinstance(module_name, string_types): - raise TypeError("Module name must be a string") + if not isinstance(locals, dict): + raise TypeError("Locals must be a dictionary") - _ast, expr = hy_compile(hytree, module_name, get_expr=True) + _ast, expr = hy_compile(hytree, module, get_expr=True) # Spoof the positions in the generated ast... for node in ast.walk(_ast): @@ -131,14 +164,13 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): if ast_callback: ast_callback(_ast, expr) - if not isinstance(namespace, dict): - raise TypeError("Globals must be a dictionary") + globals = module.__dict__ # Two-step eval: eval() the body of the exec call - eval(ast_compile(_ast, "", "exec"), namespace) + eval(ast_compile(_ast, "", "exec"), globals, locals) # Then eval the expression context and return that - return eval(ast_compile(expr, "", "eval"), namespace) + return eval(ast_compile(expr, "", "eval"), globals, locals) def cache_from_source(source_path): @@ -167,6 +199,52 @@ def cache_from_source(source_path): return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f)) +@contextmanager +def loader_module_obj(loader): + """Use the module object associated with a loader. + + This is intended to be used by a loader object itself, and primarily as a + work-around for attempts to get module and/or file code from a loader + without actually creating a module object. Since Hy currently needs the + module object for macro importing, expansion, and whatnot, using this will + reconcile Hy with such attempts. + + For example, if we're first compiling a Hy script starting from + `runpy.run_path`, the Hy compiler will need a valid module object in which + to run, but, given the way `runpy.run_path` works, there might not be one + yet (e.g. `__main__` for a .hy file). We compensate by properly loading + the module here. + + The function `inspect.getmodule` has a hidden-ish feature that returns + modules using their associated filenames (via `inspect.modulesbyfile`), + and, since the Loaders (and their delegate Loaders) carry a filename/path + associated with the parent package, we use it as a more robust attempt to + obtain an existing module object. + + When no module object is found, a temporary, minimally sufficient module + object is created for the duration of the `with` body. + """ + tmp_mod = False + + try: + module = inspect.getmodule(None, _filename=loader.path) + except KeyError: + module = None + + if module is None: + tmp_mod = True + module = sys.modules.setdefault(loader.name, + types.ModuleType(loader.name)) + module.__file__ = loader.path + module.__name__ = loader.name + + try: + yield module + finally: + if tmp_mod: + del sys.modules[loader.name] + + def _hy_code_from_file(filename, loader_type=None): """Use PEP-302 loader to produce code for a given Hy source file.""" full_fname = os.path.abspath(filename) @@ -226,7 +304,8 @@ if PY3: source = data.decode("utf-8") try: hy_tree = hy_parse(source) - data = hy_compile(hy_tree, self.name) + with loader_module_obj(self) as module: + data = hy_compile(hy_tree, module) except (HyTypeError, LexException) as e: if e.source is None: e.source = source @@ -276,6 +355,15 @@ else: super(HyLoader, self).__init__(fullname, fileobj, filename, etc) + def __getattr__(self, item): + # We add these for Python >= 3.4 Loader interface compatibility. + if item == 'path': + return self.filename + elif item == 'name': + return self.fullname + else: + return super(HyLoader, self).__getattr__(item) + def exec_module(self, module, fullname=None): fullname = self._fix_name(fullname) code = self.get_code(fullname) @@ -283,7 +371,7 @@ else: def load_module(self, fullname=None): """Same as `pkgutil.ImpLoader`, with an extra check for Hy - source""" + source and the option to not run `self.exec_module`.""" fullname = self._fix_name(fullname) ext_type = self.etc[0] mod_type = self.etc[2] @@ -298,7 +386,7 @@ else: mod = sys.modules[fullname] else: mod = sys.modules.setdefault( - fullname, imp.new_module(fullname)) + fullname, types.ModuleType(fullname)) # TODO: Should we set these only when not in `sys.modules`? if mod_type == imp.PKG_DIRECTORY: @@ -351,7 +439,8 @@ else: try: hy_source = self.get_source(fullname) hy_tree = hy_parse(hy_source) - hy_ast = hy_compile(hy_tree, fullname) + with loader_module_obj(self) as module: + hy_ast = hy_compile(hy_tree, module) code = compile(hy_ast, self.filename, 'exec', hy_ast_compile_flags) @@ -363,7 +452,7 @@ else: if not sys.dont_write_bytecode: try: - hyc_compile(code) + hyc_compile(code, module=fullname) except IOError: pass return code @@ -470,7 +559,8 @@ else: _py_compile_compile = py_compile.compile - def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False): + def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False, + module=None): """Write a Hy file, or code object, to pyc. This is a patched version of Python 2.7's `py_compile.compile`. @@ -489,6 +579,9 @@ else: The filename to use for compile-time errors. doraise : bool, default False If `True` raise compilation exceptions; otherwise, ignore them. + module : str or types.ModuleType, optional + The module, or module name, in which the Hy tree is expanded. + Default is the caller's module. Returns ------- @@ -510,7 +603,13 @@ else: flags = None if _could_be_hy_src(filename): hy_tree = hy_parse(source_str) - source = hy_compile(hy_tree, '') + + if module is None: + module = inspect.getmodule(inspect.stack()[1][0]) + elif not inspect.ismodule(module): + module = importlib.import_module(module) + + source = hy_compile(hy_tree, module) flags = hy_ast_compile_flags codeobject = compile(source, dfile or filename, 'exec', flags) diff --git a/hy/inspect.py b/hy/inspect.py deleted file mode 100644 index ff972ec..0000000 --- a/hy/inspect.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2018 the authors. -# This file is part of Hy, which is free software licensed under the Expat -# license. See the LICENSE. - -from __future__ import absolute_import - -import inspect - -try: - # Check if we have the newer inspect.signature available. - # Otherwise fallback to the legacy getargspec. - inspect.signature # noqa -except AttributeError: - def get_arity(fn): - return len(inspect.getargspec(fn)[0]) - - def has_kwargs(fn): - argspec = inspect.getargspec(fn) - return argspec.keywords is not None - - def format_args(fn): - argspec = inspect.getargspec(fn) - return inspect.formatargspec(*argspec) - -else: - def get_arity(fn): - parameters = inspect.signature(fn).parameters - return sum(1 for param in parameters.values() - if param.kind == param.POSITIONAL_OR_KEYWORD) - - def has_kwargs(fn): - parameters = inspect.signature(fn).parameters - return any(param.kind == param.VAR_KEYWORD - for param in parameters.values()) - - def format_args(fn): - return str(inspect.signature(fn)) diff --git a/hy/macros.py b/hy/macros.py index a86f043..0a6bc1b 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -1,19 +1,39 @@ # Copyright 2018 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. -import pkgutil import importlib +import inspect +import pkgutil -from collections import defaultdict - -from hy._compat import PY3 -import hy.inspect +from hy._compat import PY3, string_types from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value from hy.lex import mangle -from hy._compat import str_type from hy.errors import HyTypeError, HyMacroExpansionError +try: + # Check if we have the newer inspect.signature available. + # Otherwise fallback to the legacy getargspec. + inspect.signature # noqa +except AttributeError: + def has_kwargs(fn): + argspec = inspect.getargspec(fn) + return argspec.keywords is not None + + def format_args(fn): + argspec = inspect.getargspec(fn) + return inspect.formatargspec(*argspec) + +else: + def has_kwargs(fn): + parameters = inspect.signature(fn).parameters + return any(param.kind == param.VAR_KEYWORD + for param in parameters.values()) + + def format_args(fn): + return str(inspect.signature(fn)) + + CORE_MACROS = [ "hy.core.bootstrap", ] @@ -22,114 +42,204 @@ EXTRA_MACROS = [ "hy.core.macros", ] -_hy_macros = defaultdict(dict) -_hy_tag = defaultdict(dict) - def macro(name): """Decorator to define a macro called `name`. - - This stores the macro `name` in the namespace for the module where it is - defined. - - If the module where it is defined is in `hy.core`, then the macro is stored - in the default `None` namespace. - - This function is called from the `defmacro` special form in the compiler. - """ name = mangle(name) def _(fn): fn.__name__ = '({})'.format(name) try: - fn._hy_macro_pass_compiler = hy.inspect.has_kwargs(fn) + fn._hy_macro_pass_compiler = has_kwargs(fn) except Exception: # An exception might be raised if fn has arguments with # names that are invalid in Python. fn._hy_macro_pass_compiler = False - module_name = fn.__module__ - if module_name.startswith("hy.core"): - module_name = None - _hy_macros[module_name][name] = fn + module = inspect.getmodule(fn) + module_macros = module.__dict__.setdefault('__macros__', {}) + module_macros[name] = fn + return fn return _ def tag(name): """Decorator to define a tag macro called `name`. - - This stores the macro `name` in the namespace for the module where it is - defined. - - If the module where it is defined is in `hy.core`, then the macro is stored - in the default `None` namespace. - - This function is called from the `deftag` special form in the compiler. - """ def _(fn): _name = mangle('#{}'.format(name)) + if not PY3: _name = _name.encode('UTF-8') + fn.__name__ = _name - module_name = fn.__module__ + + module = inspect.getmodule(fn) + + module_name = module.__name__ if module_name.startswith("hy.core"): module_name = None - _hy_tag[module_name][mangle(name)] = fn + + module_tags = module.__dict__.setdefault('__tags__', {}) + module_tags[mangle(name)] = fn return fn return _ +def _same_modules(source_module, target_module): + """Compare the filenames associated with the given modules names. + + This tries to not actually load the modules. + """ + if not (source_module or target_module): + return False + + if target_module == source_module: + return True + + def _get_filename(module): + filename = None + try: + if not inspect.ismodule(module): + loader = pkgutil.get_loader(module) + if loader: + filename = loader.get_filename() + else: + filename = inspect.getfile(module) + except (TypeError, ImportError): + pass + + return filename + + source_filename = _get_filename(source_module) + target_filename = _get_filename(target_module) + + return (source_filename and target_filename and + source_filename == target_filename) + + def require(source_module, target_module, assignments, prefix=""): - """Load macros from `source_module` in the namespace of - `target_module`. `assignments` maps old names to new names, or - should be the string "ALL". If `prefix` is nonempty, it is - prepended to the name of each imported macro. (This means you get - macros named things like "mymacromodule.mymacro", which looks like - an attribute of a module, although it's actually just a symbol - with a period in its name.) + """Load macros from one module into the namespace of another. This function is called from the `require` special form in the compiler. + Parameters + ---------- + source_module: str or types.ModuleType + The module from which macros are to be imported. + + target_module: str, types.ModuleType or None + The module into which the macros will be loaded. If `None`, then + the caller's namespace. + The latter is useful during evaluation of generated AST/bytecode. + + assignments: str or list of tuples of strs + The string "ALL" or a list of macro name and alias pairs. + + prefix: str, optional ("") + If nonempty, its value is prepended to the name of each imported macro. + This allows one to emulate namespaced macros, like + "mymacromodule.mymacro", which looks like an attribute of a module. + + Returns + ------- + out: boolean + Whether or not macros and tags were actually transferred. """ - seen_names = set() + if target_module is None: + parent_frame = inspect.stack()[1][0] + target_namespace = parent_frame.f_globals + target_module = target_namespace.get('__name__', None) + elif isinstance(target_module, string_types): + target_module = importlib.import_module(target_module) + target_namespace = target_module.__dict__ + elif inspect.ismodule(target_module): + target_namespace = target_module.__dict__ + else: + raise TypeError('`target_module` is not a recognized type: {}'.format( + type(target_module))) + + # Let's do a quick check to make sure the source module isn't actually + # the module being compiled (e.g. when `runpy` executes a module's code + # in `__main__`). + # We use the module's underlying filename for this (when they exist), since + # it's the most "fixed" attribute. + if _same_modules(source_module, target_module): + return False + + if not inspect.ismodule(source_module): + source_module = importlib.import_module(source_module) + + source_macros = source_module.__dict__.setdefault('__macros__', {}) + source_tags = source_module.__dict__.setdefault('__tags__', {}) + + if len(source_module.__macros__) + len(source_module.__tags__) == 0: + if assignments != "ALL": + raise ImportError('The module {} has no macros or tags'.format( + source_module)) + else: + return False + + target_macros = target_namespace.setdefault('__macros__', {}) + target_tags = target_namespace.setdefault('__tags__', {}) + if prefix: prefix += "." - if assignments != "ALL": - assignments = {mangle(str_type(k)): v for k, v in assignments} - for d in _hy_macros, _hy_tag: - for name, macro in d[source_module].items(): - seen_names.add(name) - if assignments == "ALL": - d[target_module][mangle(prefix + name)] = macro - elif name in assignments: - d[target_module][mangle(prefix + assignments[name])] = macro + if assignments == "ALL": + # Only add macros/tags created in/by the source module. + name_assigns = [(n, n) for n, f in source_macros.items() + if inspect.getmodule(f) == source_module] + name_assigns += [(n, n) for n, f in source_tags.items() + if inspect.getmodule(f) == source_module] + else: + # If one specifically requests a macro/tag not created in the source + # module, I guess we allow it? + name_assigns = assignments - if assignments != "ALL": - unseen = frozenset(assignments.keys()).difference(seen_names) - if unseen: - raise ImportError("cannot require names: " + repr(list(unseen))) + for name, alias in name_assigns: + _name = mangle(name) + alias = mangle(prefix + alias) + if _name in source_module.__macros__: + target_macros[alias] = source_macros[_name] + elif _name in source_module.__tags__: + target_tags[alias] = source_tags[_name] + else: + raise ImportError('Could not require name {} from {}'.format( + _name, source_module)) + + return True -def load_macros(module_name): +def load_macros(module): """Load the hy builtin macros for module `module_name`. Modules from `hy.core` can only use the macros from CORE_MACROS. Other modules get the macros from CORE_MACROS and EXTRA_MACROS. - """ - for module in CORE_MACROS: - importlib.import_module(module) + builtin_macros = CORE_MACROS - if module_name.startswith("hy.core"): - return + if not module.__name__.startswith("hy.core"): + builtin_macros += EXTRA_MACROS - for module in EXTRA_MACROS: - importlib.import_module(module) + module_macros = module.__dict__.setdefault('__macros__', {}) + module_tags = module.__dict__.setdefault('__tags__', {}) + + for builtin_mod_name in builtin_macros: + builtin_mod = importlib.import_module(builtin_mod_name) + + # Make sure we don't overwrite macros in the module. + if hasattr(builtin_mod, '__macros__'): + module_macros.update({k: v + for k, v in builtin_mod.__macros__.items() + if k not in module_macros}) + if hasattr(builtin_mod, '__tags__'): + module_tags.update({k: v + for k, v in builtin_mod.__tags__.items() + if k not in module_tags}) def make_empty_fn_copy(fn): @@ -139,7 +249,7 @@ def make_empty_fn_copy(fn): # can continue running. Unfortunately, the error message that might get # raised later on while expanding a macro might not make sense at all. - formatted_args = hy.inspect.format_args(fn) + formatted_args = format_args(fn) fn_str = 'lambda {}: None'.format( formatted_args.lstrip('(').rstrip(')')) empty_fn = eval(fn_str) @@ -152,14 +262,45 @@ def make_empty_fn_copy(fn): return empty_fn -def macroexpand(tree, compiler, once=False): - """Expand the toplevel macros for the `tree`. +def macroexpand(tree, module, compiler=None, once=False): + """Expand the toplevel macros for the given Hy AST tree. - Load the macros from the given `compiler.module_name`, then expand the - (top-level) macros in `tree` until we no longer can. + Load the macros from the given `module`, then expand the (top-level) macros + in `tree` until we no longer can. + `HyExpression` resulting from macro expansions are assigned the module in + which the macro function is defined (determined using `inspect.getmodule`). + If the resulting `HyExpression` is itself macro expanded, then the + namespace of the assigned module is checked first for a macro corresponding + to the expression's head/car symbol. If the head/car symbol of such a + `HyExpression` is not found among the macros of its assigned module's + namespace, the outer-most namespace--e.g. the one given by the `module` + parameter--is used as a fallback. + + Parameters + ---------- + tree: HyObject or list + Hy AST tree. + + module: str or types.ModuleType + Module used to determine the local namespace for macros. + + compiler: HyASTCompiler, optional + The compiler object passed to expanded macros. + + once: boolean, optional + Only expand the first macro in `tree`. + + Returns + ------ + out: HyObject + Returns a mutated tree with macros expanded. """ - load_macros(compiler.module_name) + if not inspect.ismodule(module): + module = importlib.import_module(module) + + assert not compiler or compiler.module == module + while True: if not isinstance(tree, HyExpression) or tree == []: @@ -170,24 +311,34 @@ def macroexpand(tree, compiler, once=False): break fn = mangle(fn) - m = _hy_macros[compiler.module_name].get(fn) or _hy_macros[None].get(fn) + expr_modules = (([] if not hasattr(tree, 'module') else [tree.module]) + + [module]) + + # Choose the first namespace with the macro. + m = next((mod.__macros__[fn] + for mod in expr_modules + if fn in mod.__macros__), + None) if not m: break opts = {} if m._hy_macro_pass_compiler: + if compiler is None: + from hy.compiler import HyASTCompiler + compiler = HyASTCompiler(module) opts['compiler'] = compiler try: m_copy = make_empty_fn_copy(m) - m_copy(compiler.module_name, *tree[1:], **opts) + m_copy(module.__name__, *tree[1:], **opts) except TypeError as e: msg = "expanding `" + str(tree[0]) + "': " msg += str(e).replace("()", "", 1).strip() raise HyMacroExpansionError(tree, msg) try: - obj = m(compiler.module_name, *tree[1:], **opts) + obj = m(module.__name__, *tree[1:], **opts) except HyTypeError as e: if e.expression is None: e.expression = tree @@ -195,6 +346,10 @@ def macroexpand(tree, compiler, once=False): except Exception as e: msg = "expanding `" + str(tree[0]) + "': " + repr(e) raise HyMacroExpansionError(tree, msg) + + if isinstance(obj, HyExpression): + obj.module = inspect.getmodule(m) + tree = replace_hy_obj(obj, tree) if once: @@ -203,25 +358,33 @@ def macroexpand(tree, compiler, once=False): tree = wrap_value(tree) return tree -def macroexpand_1(tree, compiler): + +def macroexpand_1(tree, module, compiler=None): """Expand the toplevel macro from `tree` once, in the context of `compiler`.""" - return macroexpand(tree, compiler, once=True) + return macroexpand(tree, module, compiler, once=True) -def tag_macroexpand(tag, tree, compiler): - """Expand the tag macro "tag" with argument `tree`.""" - load_macros(compiler.module_name) +def tag_macroexpand(tag, tree, module): + """Expand the tag macro `tag` with argument `tree`.""" + if not inspect.ismodule(module): + module = importlib.import_module(module) + + expr_modules = (([] if not hasattr(tree, 'module') else [tree.module]) + + [module]) + + # Choose the first namespace with the macro. + tag_macro = next((mod.__tags__[tag] + for mod in expr_modules + if tag in mod.__tags__), + None) - tag_macro = _hy_tag[compiler.module_name].get(tag) if tag_macro is None: - try: - tag_macro = _hy_tag[None][tag] - except KeyError: - raise HyTypeError( - tag, - "`{0}' is not a defined tag macro.".format(tag) - ) + raise HyTypeError(tag, "'{0}' is not a defined tag macro.".format(tag)) expr = tag_macro(tree) + + if isinstance(expr, HyExpression): + expr.module = inspect.getmodule(tag_macro) + return replace_hy_obj(expr, tree) diff --git a/hy/models.py b/hy/models.py index 943458e..2ff5efa 100644 --- a/hy/models.py +++ b/hy/models.py @@ -32,11 +32,12 @@ class HyObject(object): Generic Hy Object model. This is helpful to inject things into all the Hy lexing Objects at once. """ + __properties__ = ["module", "start_line", "end_line", "start_column", + "end_column"] def replace(self, other, recursive=False): if isinstance(other, HyObject): - for attr in ["start_line", "end_line", - "start_column", "end_column"]: + for attr in self.__properties__: if not hasattr(self, attr) and hasattr(other, attr): setattr(self, attr, getattr(other, attr)) else: diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 224670e..d33bfea 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -56,7 +56,7 @@ def test_runpy(): def test_stringer(): - _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '') + _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '__main__') assert type(_ast.body[0]) == ast.FunctionDef @@ -80,7 +80,7 @@ def test_import_error_reporting(): def _import_error_test(): try: - _ = hy_compile(hy_parse("(import \"sys\")"), '') + _ = hy_compile(hy_parse("(import \"sys\")"), '__main__') except HyTypeError: return "Error reported" diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py index 407d756..8644532 100644 --- a/tests/macros/test_macro_processor.py +++ b/tests/macros/test_macro_processor.py @@ -22,6 +22,7 @@ def tmac(ETname, *tree): def test_preprocessor_simple(): """ Test basic macro expansion """ obj = macroexpand(tokenize('(test "one" "two")')[0], + __name__, HyASTCompiler(__name__)) assert obj == HyList(["one", "two"]) assert type(obj) == HyList @@ -30,6 +31,7 @@ def test_preprocessor_simple(): def test_preprocessor_expression(): """ Test that macro expansion doesn't recurse""" obj = macroexpand(tokenize('(test (test "one" "two"))')[0], + __name__, HyASTCompiler(__name__)) assert type(obj) == HyList @@ -41,13 +43,13 @@ def test_preprocessor_expression(): obj = HyList([HyString("one"), HyString("two")]) obj = tokenize('(shill ["one" "two"])')[0][1] - assert obj == macroexpand(obj, HyASTCompiler("")) + assert obj == macroexpand(obj, __name__, HyASTCompiler(__name__)) def test_preprocessor_exceptions(): """ Test that macro expansion raises appropriate exceptions""" with pytest.raises(HyMacroExpansionError) as excinfo: - macroexpand(tokenize('(defn)')[0], HyASTCompiler(__name__)) + macroexpand(tokenize('(defn)')[0], __name__, HyASTCompiler(__name__)) assert "_hy_anon_fn_" not in excinfo.value.message assert "TypeError" not in excinfo.value.message @@ -56,6 +58,6 @@ def test_macroexpand_nan(): # https://github.com/hylang/hy/issues/1574 import math NaN = float('nan') - x = macroexpand(HyFloat(NaN), HyASTCompiler(__name__)) + x = macroexpand(HyFloat(NaN), __name__, HyASTCompiler(__name__)) assert type(x) is HyFloat assert math.isnan(x) diff --git a/tests/macros/test_tag_macros.py b/tests/macros/test_tag_macros.py index 3cbfc94..f9c4f69 100644 --- a/tests/macros/test_tag_macros.py +++ b/tests/macros/test_tag_macros.py @@ -11,6 +11,7 @@ def test_tag_macro_error(): """Check if we get correct error with wrong dispatch character""" try: macroexpand(tokenize("(dispatch_tag_macro '- '())")[0], + __name__, HyASTCompiler(__name__)) except HyTypeError as e: assert "with the character `-`" in str(e) diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index ef6ae26..73b433b 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -2,7 +2,8 @@ ;; This file is part of Hy, which is free software licensed under the Expat ;; license. See the LICENSE. -(import [hy.errors [HyTypeError]]) +(import pytest + [hy.errors [HyTypeError]]) (defmacro rev [&rest body] "Execute the `body` statements in reverse" @@ -329,3 +330,124 @@ (except [e SystemExit] (assert (= (str e) "42")))) (setv --name-- oldname)) + +(defn test-macro-namespace-resolution [] + "Confirm that local versions of macro-macro dependencies do not shadow the +versions from the macro's own module, but do resolve unbound macro references +in expansions." + + ;; `nonlocal-test-macro` is a macro used within + ;; `tests.resources.macro-with-require.test-module-macro`. + ;; Here, we introduce an equivalently named version in local scope that, when + ;; used, will expand to a different output string. + (defmacro nonlocal-test-macro [x] + (print "this is the local version of `nonlocal-test-macro`!")) + + ;; Was the above macro created properly? + (assert (in "nonlocal_test_macro" __macros__)) + + (setv nonlocal-test-macro (get __macros__ "nonlocal_test_macro")) + + (require [tests.resources.macro-with-require [*]]) + + ;; Make sure our local version wasn't overwritten by a faulty `require` of the + ;; one in tests.resources.macro-with-require. + (assert (= nonlocal-test-macro (get __macros__ "nonlocal_test_macro"))) + + (setv module-name-var "tests.native_tests.native_macros.test-macro-namespace-resolution") + (assert (= (+ "This macro was created in tests.resources.macros, " + "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution " + "and passed the value 2.") + (test-module-macro 2))) + (assert (= (+ "This macro was created in tests.resources.macros, " + "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution " + "and passed the value 2.") + #test-module-tag 2)) + + ;; Now, let's use a `require`d macro that depends on another macro defined only + ;; in this scope. + (defmacro local-test-macro [x] + (.format "This is the local version of `nonlocal-test-macro` returning {}!" x)) + + (assert (= "This is the local version of `nonlocal-test-macro` returning 3!" + (test-module-macro-2 3))) + (assert (= "This is the local version of `nonlocal-test-macro` returning 3!" + #test-module-tag-2 3))) + +(defn test-macro-from-module [] + "Macros loaded from an external module, which itself `require`s macros, should + work without having to `require` the module's macro dependencies (due to + [minimal] macro namespace resolution). + + In doing so we also confirm that a module's `__macros__` attribute is correctly + loaded and used. + + Additionally, we confirm that `require` statements are executed via loaded bytecode." + + (import os sys marshal types) + (import [hy.importer [cache-from-source]]) + + (setv pyc-file (cache-from-source + (os.path.realpath + (os.path.join + "tests" "resources" "macro_with_require.hy")))) + + ;; Remove any cached byte-code, so that this runs from source and + ;; gets evaluated in this module. + (when (os.path.isfile pyc-file) + (os.unlink pyc-file) + (.clear sys.path_importer_cache) + (when (in "tests.resources.macro_with_require" sys.modules) + (del (get sys.modules "tests.resources.macro_with_require")) + (__macros__.clear) + (__tags__.clear))) + + ;; Ensure that bytecode isn't present when we require this module. + (assert (not (os.path.isfile pyc-file))) + + (defn test-requires-and-macros [] + (require [tests.resources.macro-with-require + [test-module-macro]]) + + ;; Make sure that `require` didn't add any of its `require`s + (assert (not (in "nonlocal-test-macro" __macros__))) + ;; and that it didn't add its tags. + (assert (not (in "test_module_tag" __tags__))) + + ;; Now, require everything. + (require [tests.resources.macro-with-require [*]]) + + ;; Again, make sure it didn't add its required macros and/or tags. + (assert (not (in "nonlocal-test-macro" __macros__))) + + ;; Its tag(s) should be here now. + (assert (in "test_module_tag" __tags__)) + + ;; The test macro expands to include this symbol. + (setv module-name-var "tests.native_tests.native_macros") + (assert (= (+ "This macro was created in tests.resources.macros, " + "expanded in tests.native_tests.native_macros " + "and passed the value 1.") + (test-module-macro 1))) + + (assert (= (+ "This macro was created in tests.resources.macros, " + "expanded in tests.native_tests.native_macros " + "and passed the value 1.") + #test-module-tag 1))) + + (test-requires-and-macros) + + ;; Now that bytecode is present, reload the module, clear the `require`d + ;; macros and tags, and rerun the tests. + (assert (os.path.isfile pyc-file)) + + ;; Reload the module and clear the local macro context. + (.clear sys.path_importer_cache) + (del (get sys.modules "tests.resources.macro_with_require")) + (.clear __macros__) + (.clear __tags__) + + ;; XXX: There doesn't seem to be a way--via standard import mechanisms--to + ;; ensure that an imported module used the cached bytecode. We'll simply have + ;; to trust that the .pyc loading convention was followed. + (test-requires-and-macros)) diff --git a/tests/resources/bin/circular_macro_require.hy b/tests/resources/bin/circular_macro_require.hy new file mode 100644 index 0000000..62d0ce2 --- /dev/null +++ b/tests/resources/bin/circular_macro_require.hy @@ -0,0 +1,8 @@ +(defmacro bar [expr] + `(print ~expr)) + +(defmacro foo [expr] + `(do (require [tests.resources.bin.circular-macro-require [bar]]) + (bar ~expr))) + +(foo 42) diff --git a/tests/resources/bin/require_and_eval.hy b/tests/resources/bin/require_and_eval.hy new file mode 100644 index 0000000..4e1c144 --- /dev/null +++ b/tests/resources/bin/require_and_eval.hy @@ -0,0 +1,3 @@ +(require [hy.extra.anaphoric [ap-if]]) + +(print (eval '(ap-if (+ "a" "b") (+ it "c")))) diff --git a/tests/resources/macro_with_require.hy b/tests/resources/macro_with_require.hy new file mode 100644 index 0000000..ef25018 --- /dev/null +++ b/tests/resources/macro_with_require.hy @@ -0,0 +1,25 @@ +;; Require all the macros and make sure they don't pollute namespaces/modules +;; that require `*` from this. +(require [tests.resources.macros [*]]) + +(defmacro test-module-macro [a] + "The variable `macro-level-var' here should not bind to the same-named symbol +in the expansion of `nonlocal-test-macro'." + (setv macro-level-var "tests.resources.macros.macro-with-require") + `(nonlocal-test-macro ~a)) + +(deftag test-module-tag [a] + "The variable `macro-level-var' here should not bind to the same-named symbol +in the expansion of `nonlocal-test-macro'." + (setv macro-level-var "tests.resources.macros.macro-with-require") + `(nonlocal-test-macro ~a)) + +(defmacro test-module-macro-2 [a] + "The macro `local-test-macro` isn't in this module's namespace, so it better + be in the expansion's!" + `(local-test-macro ~a)) + +(deftag test-module-tag-2 [a] + "The macro `local-test-macro` isn't in this module's namespace, so it better + be in the expansion's!" + `(local-test-macro ~a)) diff --git a/tests/resources/macros.hy b/tests/resources/macros.hy index 300a790..4a17a05 100644 --- a/tests/resources/macros.hy +++ b/tests/resources/macros.hy @@ -1,3 +1,5 @@ +(setv module-name-var "tests.resources.macros") + (defmacro thread-set-ab [] (defn f [&rest args] (.join "" (+ (, "a") args))) (setv variable (HySymbol (-> "b" (f)))) @@ -10,3 +12,11 @@ (defmacro test-macro [] '(setv blah 1)) + +(defmacro nonlocal-test-macro [x] + "When called from `macro-with-require`'s macro(s), the first instance of +`module-name-var` should resolve to the value in the module where this is +defined, then the expansion namespace/module" + `(.format (+ "This macro was created in {}, expanded in {} " + "and passed the value {}.") + ~module-name-var module-name-var ~x)) \ No newline at end of file diff --git a/tests/test_bin.py b/tests/test_bin.py index 58d1448..6d1e10e 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -388,3 +388,38 @@ def test_bin_hy_file_no_extension(): """Confirm that a file with no extension is processed as Hy source""" output, _ = run_cmd("hy tests/resources/no_extension") assert "This Should Still Work" in output + + +def test_bin_hy_circular_macro_require(): + """Confirm that macros can require themselves during expansion and when + run from the command line.""" + + # First, with no bytecode + test_file = "tests/resources/bin/circular_macro_require.hy" + rm(cache_from_source(test_file)) + assert not os.path.exists(cache_from_source(test_file)) + output, _ = run_cmd("hy {}".format(test_file)) + assert "42" == output.strip() + + # Now, with bytecode + assert os.path.exists(cache_from_source(test_file)) + output, _ = run_cmd("hy {}".format(test_file)) + assert "42" == output.strip() + +def test_bin_hy_macro_require(): + """Confirm that a `require` will load macros into the non-module namespace + (i.e. `exec(code, locals)`) used by `runpy.run_path`. + In other words, this confirms that the AST generated for a `require` will + load macros into the unnamed namespace its run in.""" + + # First, with no bytecode + test_file = "tests/resources/bin/require_and_eval.hy" + rm(cache_from_source(test_file)) + assert not os.path.exists(cache_from_source(test_file)) + output, _ = run_cmd("hy {}".format(test_file)) + assert "abc" == output.strip() + + # Now, with bytecode + assert os.path.exists(cache_from_source(test_file)) + output, _ = run_cmd("hy {}".format(test_file)) + assert "abc" == output.strip()