diff --git a/hy/__init__.py b/hy/__init__.py index d13c35a..f188b64 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -12,5 +12,5 @@ import hy.importer # NOQA # we import for side-effects. -from hy.core.language import read, read_str, mangle, unmangle # NOQA -from hy.importer import hy_eval as eval # NOQA +from hy.lex import read, read_str, mangle, unmangle # NOQA +from hy.compiler import hy_eval as eval # NOQA diff --git a/hy/cmdline.py b/hy/cmdline.py index eeb5ccd..7fdfb49 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -18,9 +18,10 @@ import types import astor.code_gen import hy -from hy.lex import LexException, PrematureEndOfInput, mangle -from hy.compiler import HyTypeError, hy_compile -from hy.importer import hy_eval, hy_parse, runhy +from hy.lex import hy_parse, mangle +from hy.lex.exceptions import LexException, PrematureEndOfInput +from hy.compiler import HyTypeError, hy_compile, hy_eval +from hy.importer import runhy from hy.completer import completion, Completer from hy.macros import macro, require from hy.models import HyExpression, HyString, HySymbol diff --git a/hy/compiler.py b/hy/compiler.py index f6cf052..11473d7 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -16,7 +16,8 @@ from hy.lex import mangle, unmangle from hy._compat import (str_type, string_types, bytes_type, long_type, PY3, PY35, raise_empty) from hy.macros import require, load_macros, macroexpand, tag_macroexpand -import hy.importer + +import hy.core import traceback import importlib @@ -26,6 +27,7 @@ import types import ast import sys import copy +import __future__ from collections import defaultdict @@ -37,6 +39,60 @@ else: Inf = float('inf') +hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION | + __future__.CO_FUTURE_PRINT_FUNCTION) + + +def ast_compile(ast, filename, mode): + """Compile AST. + + Parameters + ---------- + ast : instance of `ast.AST` + + filename : str + Filename used for run-time error messages + + mode: str + `compile` mode parameter + + Returns + ------- + out : instance of `types.CodeType` + """ + return compile(ast, filename, mode, hy_ast_compile_flags) + + +def calling_module(n=1): + """Get the module calling, if available. + + As a fallback, this will import a module using the calling frame's + globals value of `__name__`. + + Parameters + ---------- + n: int, optional + The number of levels up the stack from this function call. + The default is one level up. + + Returns + ------- + out: types.ModuleType + The module at stack level `n + 1` or `None`. + """ + frame_up = inspect.stack(0)[n + 1][0] + module = inspect.getmodule(frame_up) + if module is None: + # This works for modules like `__main__` + module_name = frame_up.f_globals.get('__name__', None) + if module_name: + try: + module = importlib.import_module(module_name) + except ImportError: + pass + return module + + def ast_str(x, piecewise=False): if piecewise: return ".".join(ast_str(s) if s else "" for s in x.split(".")) @@ -301,20 +357,22 @@ class HyASTCompiler(object): self.module = module self.module_name = module.__name__ - self.can_use_stdlib = ( - not self.module_name.startswith("hy.core") - or self.module_name == "hy.core.macros") + # Hy expects these to be present, so we prep the module for Hy + # compilation. + self.module.__dict__.setdefault('__macros__', {}) + self.module.__dict__.setdefault('__tags__', {}) - # Load stdlib macros into the module namespace. - load_macros(self.module) + self.can_use_stdlib = not self.module_name.startswith("hy.core") self._stdlib = {} # Everything in core needs to be explicit (except for # the core macros, which are built with the core functions). if self.can_use_stdlib: + # Load stdlib macros into the module namespace. + load_macros(self.module) + # Populate _stdlib. - import hy.core for stdlib_module in hy.core.STDLIB: mod = importlib.import_module(stdlib_module) for e in map(ast_str, getattr(mod, 'EXPORTS', [])): @@ -1554,9 +1612,7 @@ class HyASTCompiler(object): def compile_eval_and_compile(self, expr, root, body): new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr) - hy.importer.hy_eval(new_expr + body, - self.module.__dict__, - self.module) + hy_eval(new_expr + body, self.module.__dict__, self.module) return (self._compile_branch(body) if ast_str(root) == "eval_and_compile" @@ -1723,6 +1779,83 @@ class HyASTCompiler(object): return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2]) +def hy_eval(hytree, locals=None, module=None, ast_callback=None): + """Evaluates a quoted expression and returns the value. + + Examples + -------- + + => (eval '(print "Hello World")) + "Hello World" + + If you want to evaluate a string, use ``read-str`` to convert it to a + form first: + + => (eval (read-str "(+ 1 1)")) + 2 + + Parameters + ---------- + hytree: a Hy expression tree + Source code to parse. + + locals: dict, optional + Local environment in which to evaluate the Hy tree. Defaults to the + calling frame. + + module: str or types.ModuleType, optional + Module, or name of the module, to which the Hy tree is assigned and + the global values are taken. + Defaults to the calling frame's module, if any, and '__eval__' + otherwise. + + ast_callback: callable, optional + A callback that is passed the Hy compiled tree and resulting + expression object, in that order, after compilation but before + evaluation. + + Returns + ------- + out : Result of evaluating the Hy compiled tree. + """ + if module is None: + module = calling_module() + + if isinstance(module, string_types): + module = importlib.import_module(ast_str(module, piecewise=True)) + elif not inspect.ismodule(module): + raise TypeError('Invalid module type: {}'.format(type(module))) + + if locals is None: + frame = inspect.stack()[1][0] + locals = inspect.getargvalues(frame).locals + + if not isinstance(locals, dict): + raise TypeError("Locals must be a dictionary") + + _ast, expr = hy_compile(hytree, module, get_expr=True) + + # Spoof the positions in the generated ast... + for node in ast.walk(_ast): + node.lineno = 1 + node.col_offset = 1 + + for node in ast.walk(expr): + node.lineno = 1 + node.col_offset = 1 + + if ast_callback: + ast_callback(_ast, expr) + + globals = module.__dict__ + + # Two-step eval: eval() the body of the exec call + eval(ast_compile(_ast, "", "exec"), globals, locals) + + # Then eval the expression context and return that + return eval(ast_compile(expr, "", "eval"), globals, locals) + + def hy_compile(tree, module, root=ast.Module, get_expr=False): """ Compile a Hy tree into a Python AST tree. diff --git a/hy/core/language.hy b/hy/core/language.hy index 5ada235..e13cbfa 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -11,17 +11,19 @@ (import [fractions [Fraction :as fraction]]) (import operator) ; shadow not available yet (import sys) -(if-python2 - (import [StringIO [StringIO]]) - (import [io [StringIO]])) (import [hy._compat [long-type]]) ; long for python2, int for python3 +(import [hy.models [HySymbol HyKeyword]]) +(import [hy.lex [tokenize mangle unmangle read read-str]]) +(import [hy.lex.exceptions [LexException PrematureEndOfInput]]) +(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]]) + +(import [hy.core.shadow [*]]) + +(require [hy.core.bootstrap [*]]) + (if-python2 (import [collections :as cabc]) (import [collections.abc :as cabc])) -(import [hy.models [HySymbol HyKeyword]]) -(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]]) -(import [hy.compiler [HyASTCompiler]]) -(import [hy.importer [calling-module hy-eval :as eval]]) (defn butlast [coll] "Return an iterator of all but the last item in `coll`." @@ -415,28 +417,6 @@ Raises ValueError for (not (pos? n))." "Check if `n` equals 0." (= n 0)) -(defn read [&optional [from-file sys.stdin] - [eof ""]] - "Read from input and returns a tokenized string. - -Can take a given input buffer to read from, and a single byte -as EOF (defaults to an empty string)." - (setv buff "") - (while True - (setv inn (string (.readline from-file))) - (if (= inn eof) - (raise (EOFError "Reached end of file"))) - (+= buff inn) - (try - (setv parsed (first (tokenize buff))) - (except [e [PrematureEndOfInput IndexError]]) - (else (break)))) - parsed) - -(defn read-str [input] - "Reads and tokenizes first line of `input`." - (read :from-file (StringIO input))) - (defn keyword [value] "Create a keyword from `value`. diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 2f9154e..0e6ab5f 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -8,6 +8,11 @@ (import [hy.models [HyList HySymbol]]) +(eval-and-compile + (import [hy.core.language [*]])) + +(require [hy.core.bootstrap [*]]) + (defmacro as-> [head name &rest rest] "Beginning with `head`, expand a sequence of assignments `rest` to `name`. diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy index ecf42a5..9fae8fe 100644 --- a/hy/core/shadow.hy +++ b/hy/core/shadow.hy @@ -7,6 +7,8 @@ (import operator) (import [hy._compat [PY3 PY35]]) +(require [hy.core.bootstrap [*]]) + (if PY3 (import [functools [reduce]])) diff --git a/hy/importer.py b/hy/importer.py index fdcb0be..5e68c61 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -6,7 +6,6 @@ from __future__ import absolute_import import sys import os -import ast import inspect import pkgutil import re @@ -14,163 +13,15 @@ import io import types import tempfile import importlib -import __future__ from functools import partial from contextlib import contextmanager from hy.errors import HyTypeError -from hy.compiler import hy_compile, ast_str -from hy.lex import tokenize, LexException -from hy.models import HyExpression, HySymbol -from hy._compat import string_types, PY3 - - -hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION | - __future__.CO_FUTURE_PRINT_FUNCTION) - - -def calling_module(n=1): - """Get the module calling, if available. - - As a fallback, this will import a module using the calling frame's - globals value of `__name__`. - - Parameters - ---------- - n: int, optional - The number of levels up the stack from this function call. - The default is one level up. - - Returns - ------- - out: types.ModuleType - The module at stack level `n + 1` or `None`. - """ - frame_up = inspect.stack(0)[n + 1][0] - module = inspect.getmodule(frame_up) - if module is None: - # This works for modules like `__main__` - module_name = frame_up.f_globals.get('__name__', None) - if module_name: - try: - module = importlib.import_module(module_name) - except ImportError: - pass - return module - - -def ast_compile(ast, filename, mode): - """Compile AST. - - Parameters - ---------- - ast : instance of `ast.AST` - - filename : str - Filename used for run-time error messages - - mode: str - `compile` mode parameter - - Returns - ------- - out : instance of `types.CodeType` - """ - return compile(ast, filename, mode, hy_ast_compile_flags) - - -def hy_parse(source): - """Parse a Hy source string. - - Parameters - ---------- - source: string - Source code to parse. - - Returns - ------- - out : instance of `types.CodeType` - """ - source = re.sub(r'\A#!.*', '', source) - return HyExpression([HySymbol("do")] + tokenize(source + "\n")) - - -def hy_eval(hytree, locals=None, module=None, ast_callback=None): - """Evaluates a quoted expression and returns the value. - - Examples - -------- - - => (eval '(print "Hello World")) - "Hello World" - - If you want to evaluate a string, use ``read-str`` to convert it to a - form first: - - => (eval (read-str "(+ 1 1)")) - 2 - - Parameters - ---------- - hytree: a Hy expression tree - Source code to parse. - - locals: dict, optional - Local environment in which to evaluate the Hy tree. Defaults to the - calling frame. - - module: str or types.ModuleType, optional - Module, or name of the module, to which the Hy tree is assigned and - the global values are taken. - Defaults to the calling frame's module, if any, and '__eval__' - otherwise. - - ast_callback: callable, optional - A callback that is passed the Hy compiled tree and resulting - expression object, in that order, after compilation but before - evaluation. - - Returns - ------- - out : Result of evaluating the Hy compiled tree. - """ - if module is None: - module = calling_module() - - if isinstance(module, string_types): - module = importlib.import_module(ast_str(module, piecewise=True)) - elif not inspect.ismodule(module): - raise TypeError('Invalid module type: {}'.format(type(module))) - - if locals is None: - frame = inspect.stack()[1][0] - locals = inspect.getargvalues(frame).locals - - if not isinstance(locals, dict): - raise TypeError("Locals must be a dictionary") - - _ast, expr = hy_compile(hytree, module, get_expr=True) - - # Spoof the positions in the generated ast... - for node in ast.walk(_ast): - node.lineno = 1 - node.col_offset = 1 - - for node in ast.walk(expr): - node.lineno = 1 - node.col_offset = 1 - - if ast_callback: - ast_callback(_ast, expr) - - globals = module.__dict__ - - # Two-step eval: eval() the body of the exec call - eval(ast_compile(_ast, "", "exec"), globals, locals) - - # Then eval the expression context and return that - return eval(ast_compile(expr, "", "eval"), globals, locals) +from hy.compiler import hy_compile, hy_ast_compile_flags +from hy.lex import hy_parse +from hy.lex.exceptions import LexException +from hy._compat import PY3 def cache_from_source(source_path): diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py index 5c05143..2103275 100644 --- a/hy/lex/__init__.py +++ b/hy/lex/__init__.py @@ -4,9 +4,35 @@ from __future__ import unicode_literals -import re, unicodedata +import re +import sys +import unicodedata + from hy._compat import str_type, isidentifier, UCS4 -from hy.lex.exceptions import LexException, PrematureEndOfInput # NOQA +from hy.lex.exceptions import PrematureEndOfInput, LexException # NOQA +from hy.models import HyExpression, HySymbol + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + + +def hy_parse(source): + """Parse a Hy source string. + + Parameters + ---------- + source: string + Source code to parse. + + Returns + ------- + out : instance of `types.CodeType` + """ + source = re.sub(r'\A#!.*', '', source) + return HyExpression([HySymbol("do")] + tokenize(source + "\n")) + def tokenize(buf): """ @@ -102,3 +128,28 @@ def unicode_to_ucs4iter(ustr): ucs4_list[i] += ucs4_list[i + 1] del ucs4_list[i + 1] return ucs4_list + + +def read(from_file=sys.stdin, eof=""): + """Read from input and returns a tokenized string. + + Can take a given input buffer to read from, and a single byte as EOF + (defaults to an empty string). + """ + buff = "" + while True: + inn = str(from_file.readline()) + if inn == eof: + raise EOFError("Reached end of file") + buff += inn + try: + parsed = next(iter(tokenize(buff)), None) + except (PrematureEndOfInput, IndexError): + pass + else: + break + return parsed + + +def read_str(input): + return read(StringIO(str_type(input))) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index cb3ab7b..1f94c3f 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -7,8 +7,9 @@ from __future__ import unicode_literals from hy import HyString from hy.models import HyObject -from hy.importer import hy_compile, hy_eval, hy_parse +from hy.compiler import hy_compile, hy_eval from hy.errors import HyCompileError, HyTypeError +from hy.lex import hy_parse from hy.lex.exceptions import LexException from hy._compat import PY3 diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index d33bfea..1973fe8 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -15,9 +15,10 @@ import pytest import hy from hy.errors import HyTypeError -from hy.lex import LexException +from hy.lex import hy_parse +from hy.lex.exceptions import LexException from hy.compiler import hy_compile -from hy.importer import hy_parse, HyLoader, cache_from_source +from hy.importer import HyLoader, cache_from_source try: from importlib import reload diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index 73b433b..fc17edb 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -148,7 +148,8 @@ (defn test-gensym-in-macros [] (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro nif [expr pos zero neg] (setv g (gensym)) `(do @@ -174,7 +175,8 @@ (defn test-with-gensym [] (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro nif [expr pos zero neg] (with-gensyms [a] `(do @@ -198,7 +200,8 @@ (defn test-defmacro/g! [] (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro/g! nif [expr pos zero neg] `(do (setv ~g!res ~expr) @@ -227,7 +230,8 @@ ;; defmacro! must do everything defmacro/g! can (import ast) (import [astor.code-gen [to-source]]) - (import [hy.importer [hy-parse hy-compile]]) + (import [hy.compiler [hy-compile]]) + (import [hy.lex [hy-parse]]) (setv macro1 "(defmacro! nif [expr pos zero neg] `(do (setv ~g!res ~expr) diff --git a/tests/test_lex.py b/tests/test_lex.py index 411c246..2e68876 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -5,7 +5,8 @@ from math import isnan from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol, HyString, HyDict, HyList, HySet, HyKeyword) -from hy.lex import LexException, PrematureEndOfInput, tokenize +from hy.lex import tokenize +from hy.lex.exceptions import LexException, PrematureEndOfInput import pytest def peoi(): return pytest.raises(PrematureEndOfInput)