From 86fda31ab1d559e08b0852ae7b78973be52b80d3 Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Sat, 10 Nov 2018 12:53:28 -0600 Subject: [PATCH] Move compilation and parsing functions out of `importer.py` Functions and variables relating to compilation and parsing have been moved to `compiler.py` and `lex/__init__.py`, respectively. Those functions are - `hy_parse` from `hy.importer` to `hy.lex` - `hy_eval`, `ast_compile`, and `calling_module` from `hy.importer` to `hy.compiler` Closes hylang/hy#1695. --- hy/__init__.py | 2 +- hy/cmdline.py | 7 +- hy/compiler.py | 137 +++++++++++++++++++++++- hy/core/language.hy | 6 +- hy/importer.py | 157 +--------------------------- hy/lex/__init__.py | 24 ++++- tests/compilers/test_ast.py | 3 +- tests/importer/test_importer.py | 5 +- tests/native_tests/native_macros.hy | 12 ++- tests/test_lex.py | 3 +- 10 files changed, 182 insertions(+), 174 deletions(-) diff --git a/hy/__init__.py b/hy/__init__.py index d13c35a..30248e0 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -13,4 +13,4 @@ import hy.importer # NOQA from hy.core.language import read, read_str, mangle, unmangle # NOQA -from hy.importer import hy_eval as eval # NOQA +from hy.compiler import hy_eval as eval # NOQA diff --git a/hy/cmdline.py b/hy/cmdline.py index eeb5ccd..7fdfb49 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -18,9 +18,10 @@ import types import astor.code_gen import hy -from hy.lex import LexException, PrematureEndOfInput, mangle -from hy.compiler import HyTypeError, hy_compile -from hy.importer import hy_eval, hy_parse, runhy +from hy.lex import hy_parse, mangle +from hy.lex.exceptions import LexException, PrematureEndOfInput +from hy.compiler import HyTypeError, hy_compile, hy_eval +from hy.importer import runhy from hy.completer import completion, Completer from hy.macros import macro, require from hy.models import HyExpression, HyString, HySymbol diff --git a/hy/compiler.py b/hy/compiler.py index f6cf052..2b262a1 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -16,7 +16,6 @@ from hy.lex import mangle, unmangle from hy._compat import (str_type, string_types, bytes_type, long_type, PY3, PY35, raise_empty) from hy.macros import require, load_macros, macroexpand, tag_macroexpand -import hy.importer import traceback import importlib @@ -26,6 +25,7 @@ import types import ast import sys import copy +import __future__ from collections import defaultdict @@ -37,6 +37,60 @@ else: Inf = float('inf') +hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION | + __future__.CO_FUTURE_PRINT_FUNCTION) + + +def ast_compile(ast, filename, mode): + """Compile AST. + + Parameters + ---------- + ast : instance of `ast.AST` + + filename : str + Filename used for run-time error messages + + mode: str + `compile` mode parameter + + Returns + ------- + out : instance of `types.CodeType` + """ + return compile(ast, filename, mode, hy_ast_compile_flags) + + +def calling_module(n=1): + """Get the module calling, if available. + + As a fallback, this will import a module using the calling frame's + globals value of `__name__`. + + Parameters + ---------- + n: int, optional + The number of levels up the stack from this function call. + The default is one level up. + + Returns + ------- + out: types.ModuleType + The module at stack level `n + 1` or `None`. + """ + frame_up = inspect.stack(0)[n + 1][0] + module = inspect.getmodule(frame_up) + if module is None: + # This works for modules like `__main__` + module_name = frame_up.f_globals.get('__name__', None) + if module_name: + try: + module = importlib.import_module(module_name) + except ImportError: + pass + return module + + def ast_str(x, piecewise=False): if piecewise: return ".".join(ast_str(s) if s else "" for s in x.split(".")) @@ -1554,9 +1608,7 @@ class HyASTCompiler(object): def compile_eval_and_compile(self, expr, root, body): new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr) - hy.importer.hy_eval(new_expr + body, - self.module.__dict__, - self.module) + hy_eval(new_expr + body, self.module.__dict__, self.module) return (self._compile_branch(body) if ast_str(root) == "eval_and_compile" @@ -1723,6 +1775,83 @@ class HyASTCompiler(object): return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2]) +def hy_eval(hytree, locals=None, module=None, ast_callback=None): + """Evaluates a quoted expression and returns the value. + + Examples + -------- + + => (eval '(print "Hello World")) + "Hello World" + + If you want to evaluate a string, use ``read-str`` to convert it to a + form first: + + => (eval (read-str "(+ 1 1)")) + 2 + + Parameters + ---------- + hytree: a Hy expression tree + Source code to parse. + + locals: dict, optional + Local environment in which to evaluate the Hy tree. Defaults to the + calling frame. + + module: str or types.ModuleType, optional + Module, or name of the module, to which the Hy tree is assigned and + the global values are taken. + Defaults to the calling frame's module, if any, and '__eval__' + otherwise. + + ast_callback: callable, optional + A callback that is passed the Hy compiled tree and resulting + expression object, in that order, after compilation but before + evaluation. + + Returns + ------- + out : Result of evaluating the Hy compiled tree. + """ + if module is None: + module = calling_module() + + if isinstance(module, string_types): + module = importlib.import_module(ast_str(module, piecewise=True)) + elif not inspect.ismodule(module): + raise TypeError('Invalid module type: {}'.format(type(module))) + + if locals is None: + frame = inspect.stack()[1][0] + locals = inspect.getargvalues(frame).locals + + if not isinstance(locals, dict): + raise TypeError("Locals must be a dictionary") + + _ast, expr = hy_compile(hytree, module, get_expr=True) + + # Spoof the positions in the generated ast... + for node in ast.walk(_ast): + node.lineno = 1 + node.col_offset = 1 + + for node in ast.walk(expr): + node.lineno = 1 + node.col_offset = 1 + + if ast_callback: + ast_callback(_ast, expr) + + globals = module.__dict__ + + # Two-step eval: eval() the body of the exec call + eval(ast_compile(_ast, "", "exec"), globals, locals) + + # Then eval the expression context and return that + return eval(ast_compile(expr, "", "eval"), globals, locals) + + def hy_compile(tree, module, root=ast.Module, get_expr=False): """ Compile a Hy tree into a Python AST tree. diff --git a/hy/core/language.hy b/hy/core/language.hy index 5ada235..981b3d3 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -19,9 +19,9 @@ (import [collections :as cabc]) (import [collections.abc :as cabc])) (import [hy.models [HySymbol HyKeyword]]) -(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]]) -(import [hy.compiler [HyASTCompiler]]) -(import [hy.importer [calling-module hy-eval :as eval]]) +(import [hy.lex [tokenize mangle unmangle]]) +(import [hy.lex.exceptions [LexException PrematureEndOfInput]]) +(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]]) (defn butlast [coll] "Return an iterator of all but the last item in `coll`." diff --git a/hy/importer.py b/hy/importer.py index fdcb0be..5e68c61 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -6,7 +6,6 @@ from __future__ import absolute_import import sys import os -import ast import inspect import pkgutil import re @@ -14,163 +13,15 @@ import io import types import tempfile import importlib -import __future__ from functools import partial from contextlib import contextmanager from hy.errors import HyTypeError -from hy.compiler import hy_compile, ast_str -from hy.lex import tokenize, LexException -from hy.models import HyExpression, HySymbol -from hy._compat import string_types, PY3 - - -hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION | - __future__.CO_FUTURE_PRINT_FUNCTION) - - -def calling_module(n=1): - """Get the module calling, if available. - - As a fallback, this will import a module using the calling frame's - globals value of `__name__`. - - Parameters - ---------- - n: int, optional - The number of levels up the stack from this function call. - The default is one level up. - - Returns - ------- - out: types.ModuleType - The module at stack level `n + 1` or `None`. - """ - frame_up = inspect.stack(0)[n + 1][0] - module = inspect.getmodule(frame_up) - if module is None: - # This works for modules like `__main__` - module_name = frame_up.f_globals.get('__name__', None) - if module_name: - try: - module = importlib.import_module(module_name) - except ImportError: - pass - return module - - -def ast_compile(ast, filename, mode): - """Compile AST. - - Parameters - ---------- - ast : instance of `ast.AST` - - filename : str - Filename used for run-time error messages - - mode: str - `compile` mode parameter - - Returns - ------- - out : instance of `types.CodeType` - """ - return compile(ast, filename, mode, hy_ast_compile_flags) - - -def hy_parse(source): - """Parse a Hy source string. - - Parameters - ---------- - source: string - Source code to parse. - - Returns - ------- - out : instance of `types.CodeType` - """ - source = re.sub(r'\A#!.*', '', source) - return HyExpression([HySymbol("do")] + tokenize(source + "\n")) - - -def hy_eval(hytree, locals=None, module=None, ast_callback=None): - """Evaluates a quoted expression and returns the value. - - Examples - -------- - - => (eval '(print "Hello World")) - "Hello World" - - If you want to evaluate a string, use ``read-str`` to convert it to a - form first: - - => (eval (read-str "(+ 1 1)")) - 2 - - Parameters - ---------- - hytree: a Hy expression tree - Source code to parse. - - locals: dict, optional - Local environment in which to evaluate the Hy tree. Defaults to the - calling frame. - - module: str or types.ModuleType, optional - Module, or name of the module, to which the Hy tree is assigned and - the global values are taken. - Defaults to the calling frame's module, if any, and '__eval__' - otherwise. - - ast_callback: callable, optional - A callback that is passed the Hy compiled tree and resulting - expression object, in that order, after compilation but before - evaluation. - - Returns - ------- - out : Result of evaluating the Hy compiled tree. - """ - if module is None: - module = calling_module() - - if isinstance(module, string_types): - module = importlib.import_module(ast_str(module, piecewise=True)) - elif not inspect.ismodule(module): - raise TypeError('Invalid module type: {}'.format(type(module))) - - if locals is None: - frame = inspect.stack()[1][0] - locals = inspect.getargvalues(frame).locals - - if not isinstance(locals, dict): - raise TypeError("Locals must be a dictionary") - - _ast, expr = hy_compile(hytree, module, get_expr=True) - - # Spoof the positions in the generated ast... - for node in ast.walk(_ast): - node.lineno = 1 - node.col_offset = 1 - - for node in ast.walk(expr): - node.lineno = 1 - node.col_offset = 1 - - if ast_callback: - ast_callback(_ast, expr) - - globals = module.__dict__ - - # Two-step eval: eval() the body of the exec call - eval(ast_compile(_ast, "", "exec"), globals, locals) - - # Then eval the expression context and return that - return eval(ast_compile(expr, "", "eval"), globals, locals) +from hy.compiler import hy_compile, hy_ast_compile_flags +from hy.lex import hy_parse +from hy.lex.exceptions import LexException +from hy._compat import PY3 def cache_from_source(source_path): diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py index 5c05143..093fa49 100644 --- a/hy/lex/__init__.py +++ b/hy/lex/__init__.py @@ -4,9 +4,29 @@ from __future__ import unicode_literals -import re, unicodedata +import re +import unicodedata + from hy._compat import str_type, isidentifier, UCS4 -from hy.lex.exceptions import LexException, PrematureEndOfInput # NOQA +from hy.lex.exceptions import LexException # NOQA +from hy.models import HyExpression, HySymbol + + +def hy_parse(source): + """Parse a Hy source string. + + Parameters + ---------- + source: string + Source code to parse. + + Returns + ------- + out : instance of `types.CodeType` + """ + source = re.sub(r'\A#!.*', '', source) + return HyExpression([HySymbol("do")] + tokenize(source + "\n")) + def tokenize(buf): """ diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index cb3ab7b..1f94c3f 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -7,8 +7,9 @@ from __future__ import unicode_literals from hy import HyString from hy.models import HyObject -from hy.importer import hy_compile, hy_eval, hy_parse +from hy.compiler import hy_compile, hy_eval from hy.errors import HyCompileError, HyTypeError +from hy.lex import hy_parse from hy.lex.exceptions import LexException from hy._compat import PY3 diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index d33bfea..1973fe8 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -15,9 +15,10 @@ import pytest import hy from hy.errors import HyTypeError -from hy.lex import LexException +from hy.lex import hy_parse +from hy.lex.exceptions import LexException from hy.compiler import hy_compile -from hy.importer import hy_parse, HyLoader, cache_from_source +from hy.importer import HyLoader, cache_from_source try: from importlib import reload diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index 73b433b..fc17edb 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -148,7 +148,8 @@ (defn test-gensym-in-macros [] (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro nif [expr pos zero neg] (setv g (gensym)) `(do @@ -174,7 +175,8 @@ (defn test-with-gensym [] (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro nif [expr pos zero neg] (with-gensyms [a] `(do @@ -198,7 +200,8 @@ (defn test-defmacro/g! [] (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro/g! nif [expr pos zero neg] `(do (setv ~g!res ~expr) @@ -227,7 +230,8 @@ ;; defmacro! must do everything defmacro/g! can (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro! nif [expr pos zero neg] `(do (setv ~g!res ~expr) diff --git a/tests/test_lex.py b/tests/test_lex.py index 411c246..2e68876 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -5,7 +5,8 @@ from math import isnan from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol, HyString, HyDict, HyList, HySet, HyKeyword) -from hy.lex import LexException, PrematureEndOfInput, tokenize +from hy.lex import tokenize +from hy.lex.exceptions import LexException, PrematureEndOfInput import pytest def peoi(): return pytest.raises(PrematureEndOfInput)