Merge pull request #1314 from Kodiologist/compiler-autobox

Auto-promote values to HyObjects in the compiler
This commit is contained in:
Tuukka Turto 2017-07-10 09:04:08 +03:00 committed by GitHub
commit 7c53a07b93
12 changed files with 164 additions and 109 deletions

4
NEWS
View File

@ -4,6 +4,10 @@ Changes from 0.13.0
* Single-character "sharp macros" changed to "tag macros", which can have
longer names
* Periods are no longer allowed in keywords
* `eval` is now a function instead of a special form
* The compiler now automatically promotes values to Hy model objects
as necessary, so you can write ``(eval `(+ 1 ~n))`` instead of
``(eval `(+ 1 ~(HyInteger n)))``
[ Bug Fixes ]
* Numeric literals are no longer parsed as symbols when followed by a dot

View File

@ -863,27 +863,6 @@ doto
=> collection
[2 1]
eval
----
``eval`` evaluates a quoted expression and returns the value. The optional
second and third arguments specify the dictionary of globals to use and the
module name. The globals dictionary defaults to ``(local)`` and the module name
defaults to the name of the current module.
.. code-block:: clj
=> (eval '(print "Hello World"))
"Hello World"
If you want to evaluate a string, use ``read-str`` to convert it to a
form first:
.. code-block:: clj
=> (eval (read-str "(+ 1 1)"))
2
eval-and-compile
----------------

View File

@ -230,6 +230,30 @@ Returns ``True`` if *coll* is empty. Equivalent to ``(= 0 (len coll))``.
False
.. _eval-fn:
eval
----
``eval`` evaluates a quoted expression and returns the value. The optional
second and third arguments specify the dictionary of globals to use and the
module name. The globals dictionary defaults to ``(local)`` and the module name
defaults to the name of the current module.
.. code-block:: clj
=> (eval '(print "Hello World"))
"Hello World"
If you want to evaluate a string, use ``read-str`` to convert it to a
form first:
.. code-block:: clj
=> (eval (read-str "(+ 1 1)"))
2
.. _every?-fn:
every?

View File

@ -15,7 +15,7 @@ import astor.codegen
import hy
from hy.lex import LexException, PrematureEndOfInput, tokenize
from hy.lex import LexException, PrematureEndOfInput
from hy.lex.parser import hy_symbol_mangle
from hy.compiler import HyTypeError
from hy.importer import (hy_eval, import_buffer_to_module,
@ -77,12 +77,9 @@ class HyREPL(code.InteractiveConsole):
global SIMPLE_TRACEBACKS
try:
try:
tokens = tokenize(source)
do = import_buffer_to_hst(source)
except PrematureEndOfInput:
return True
do = HyExpression([HySymbol('do')] + tokens)
do.start_line = do.end_line = do.start_column = do.end_column = 1
do.replace(do)
except LexException as e:
if e.source is None:
e.source = source

View File

@ -5,14 +5,15 @@
from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
HyString, HyBytes, HySymbol, HyFloat, HyList, HySet,
HyDict, HyCons)
HyDict, HyCons, wrap_value)
from hy.errors import HyCompileError, HyTypeError
from hy.lex.parser import hy_symbol_mangle
import hy.macros
from hy._compat import (
str_type, bytes_type, long_type, PY3, PY34, PY35, raise_empty)
str_type, string_types, bytes_type, long_type, PY3, PY34, PY35,
raise_empty)
from hy.macros import require, macroexpand, tag_macroexpand
import hy.importer
@ -110,6 +111,19 @@ def builds_if(_type, condition):
return lambda fn: fn
def spoof_positions(obj):
if not isinstance(obj, HyObject) or isinstance(obj, HyCons):
return
if not hasattr(obj, "start_column"):
obj.start_column = 0
if not hasattr(obj, "start_line"):
obj.start_line = 0
if (hasattr(obj, "__iter__") and
not isinstance(obj, (string_types, bytes_type))):
for x in obj:
spoof_positions(x)
class Result(object):
"""
Smart representation of the result of a hy->AST compilation
@ -378,23 +392,23 @@ class HyASTCompiler(object):
ret = Result()
for module, names in self.imports.items():
if None in names:
ret += self.compile([
HyExpression([
e = HyExpression([
HySymbol("import"),
HySymbol(module),
]).replace(expr)
])
spoof_positions(e)
ret += self.compile(e)
names = sorted(name for name in names if name)
if names:
ret += self.compile([
HyExpression([
e = HyExpression([
HySymbol("import"),
HyList([
HySymbol(module),
HyList([HySymbol(name) for name in names])
])
]).replace(expr)
])
spoof_positions(e)
ret += self.compile(e)
self.imports = defaultdict(set)
return ret.stmts
@ -404,6 +418,11 @@ class HyASTCompiler(object):
if not isinstance(ret, Result):
ret = Result() + ret
return ret
if not isinstance(atom, HyObject):
atom = wrap_value(atom)
if isinstance(atom, HyObject):
spoof_positions(atom)
return self.compile_atom(type(atom), atom)
def compile(self, tree):
try:
@ -602,12 +621,6 @@ class HyASTCompiler(object):
ast.copy_location(new_name, name)
return new_name
@builds(list)
def compile_raw_list(self, entries):
ret = self._compile_branch(entries)
ret += ret.expr_as_stmt()
return ret
def _render_quoted_form(self, form, level):
"""
Render a quoted form as a new HyExpression.
@ -706,31 +719,6 @@ class HyASTCompiler(object):
raise HyTypeError(expr,
"`%s' can't be used at the top-level" % expr[0])
@builds("eval")
@checkargs(min=1, max=3)
def compile_eval(self, expr):
expr.pop(0)
if not isinstance(expr[0], (HyExpression, HySymbol)):
raise HyTypeError(expr, "expression expected as first argument")
elist = [HySymbol("hy_eval")] + [expr[0]]
if len(expr) >= 2:
elist.append(expr[1])
else:
elist.append(HyExpression([HySymbol("locals")]))
if len(expr) == 3:
elist.append(expr[2])
else:
elist.append(HyString(self.module_name))
ret = self.compile(HyExpression(elist).replace(expr))
ret.add_imports("hy.importer", ["hy_eval"])
return ret
@builds("do")
def compile_do(self, expression):
expression.pop(0)
@ -766,6 +754,7 @@ class HyASTCompiler(object):
return ret
@builds("try")
@checkargs(min=2)
def compile_try_expression(self, expr):
expr.pop(0) # try
@ -1858,6 +1847,7 @@ class HyASTCompiler(object):
op = ops[expression.pop(0)]
right_associative = op == ast.Pow
lineno, col_offset = expression.start_line, expression.start_column
if right_associative:
expression = expression[::-1]
ret = self.compile(expression.pop(0))
@ -1870,8 +1860,8 @@ class HyASTCompiler(object):
ret += ast.BinOp(left=left_expr,
op=op(),
right=right_expr,
lineno=child.start_line,
col_offset=child.start_column)
lineno=lineno,
col_offset=col_offset)
return ret
@builds("**")
@ -2619,21 +2609,21 @@ def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
body = []
expr = None
if not (isinstance(tree, HyObject) or type(tree) is list):
raise HyCompileError("tree must be a HyObject or a list")
if not isinstance(tree, HyObject):
tree = wrap_value(tree)
if not isinstance(tree, HyObject):
raise HyCompileError("`tree` must be a HyObject or capable of "
"being promoted to one")
spoof_positions(tree)
if isinstance(tree, HyObject) or tree:
compiler = HyASTCompiler(module_name)
result = compiler.compile(tree)
expr = result.force_expr
compiler = HyASTCompiler(module_name)
result = compiler.compile(tree)
expr = result.force_expr
if not get_expr:
result += result.expr_as_stmt()
if not get_expr:
result += result.expr_as_stmt()
# We need to test that the type is *exactly* `list` because we don't
# want to do `tree[0]` on HyList or such.
spoof_tree = tree[0] if type(tree) is list else tree
body = compiler.imports_as_stmts(spoof_tree) + result.stmts
body = compiler.imports_as_stmts(tree) + result.stmts
ret = root(body=body)

View File

@ -18,7 +18,8 @@
(import [hy._compat [long-type]]) ; long for python2, int for python3
(import [hy.models [HyCons HySymbol HyKeyword]])
(import [hy.lex [LexException PrematureEndOfInput tokenize]])
(import [hy.compiler [HyASTCompiler]])
(import [hy.compiler [HyASTCompiler spoof-positions]])
(import [hy.importer [hy-eval :as eval]])
(defn butlast [coll]
"Returns coll except of last element."
@ -74,8 +75,8 @@
(import astor)
(import hy.compiler)
(fake-source-positions tree)
(setv compiled (hy.compiler.hy_compile tree (calling-module-name)))
(spoof-positions tree)
(setv compiled (hy.compiler.hy-compile tree (calling-module-name)))
((if codegen
astor.codegen.to_source
astor.dump)
@ -174,15 +175,6 @@
"Return true if (pred x) is logical true for every x in coll, else false"
(all (map pred coll)))
(defn fake-source-positions [tree]
"Fake the source positions for a given tree"
(if (coll? tree)
(for* [subtree tree]
(fake-source-positions subtree)))
(for* [attr '[start-line end-line start-column end-column]]
(if (not (hasattr tree attr))
(setattr tree attr 1))))
(defn flatten [coll]
"Return a single flat list expanding all members of coll"
(if (coll? coll)
@ -469,7 +461,7 @@
(def *exports*
'[*map accumulate butlast calling-module-name chain coll? combinations
comp complement compress cons cons? constantly count cycle dec distinct
disassemble drop drop-last drop-while empty? even? every? first filter
disassemble drop drop-last drop-while empty? eval even? every? first filter
flatten float? fraction gensym group-by identity inc input instance?
integer integer? integer-char? interleave interpose islice iterable?
iterate iterator? juxt keyword keyword? last list* macroexpand

View File

@ -3,7 +3,7 @@
# license. See the LICENSE.
from hy.compiler import hy_compile, HyTypeError
from hy.models import HyObject, replace_hy_obj
from hy.models import HyObject, HyExpression, HySymbol, replace_hy_obj
from hy.lex import tokenize, LexException
from hy.errors import HyIOError
@ -14,6 +14,7 @@ import struct
import imp
import sys
import ast
import inspect
import os
import __future__
@ -31,7 +32,7 @@ def ast_compile(ast, filename, mode):
def import_buffer_to_hst(buf):
"""Import content from buf and return a Hy AST."""
return tokenize(buf + "\n")
return HyExpression([HySymbol("do")] + tokenize(buf + "\n"))
def import_file_to_hst(fpath):
@ -142,7 +143,27 @@ def import_buffer_to_module(module_name, buf):
return mod
def hy_eval(hytree, namespace, module_name, ast_callback=None):
def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
"""``eval`` evaluates a quoted expression and returns the value. The optional
second and third arguments specify the dictionary of globals to use and the
module name. The globals dictionary defaults to ``(local)`` and the module
name defaults to the name of the current module.
=> (eval '(print "Hello World"))
"Hello World"
If you want to evaluate a string, use ``read-str`` to convert it to a
form first:
=> (eval (read-str "(+ 1 1)"))
2"""
if namespace is None:
frame = inspect.stack()[1][0]
namespace = inspect.getargvalues(frame).locals
if module_name is None:
m = inspect.getmodule(inspect.stack()[1][0])
module_name = '__eval__' if m is None else m.__name__
foo = HyObject()
foo.start_line = 0
foo.end_line = 0

View File

@ -3,7 +3,7 @@
# license. See the LICENSE.
from inspect import getargspec, formatargspec
from hy.models import replace_hy_obj, wrap_value, HyExpression, HyString
from hy.models import replace_hy_obj, HyExpression, HyString
from hy.errors import HyTypeError, HyMacroExpansionError
@ -196,7 +196,7 @@ def macroexpand_1(tree, compiler):
raise HyMacroExpansionError(tree, msg)
try:
obj = wrap_value(m(*ntree[1:], **opts))
obj = m(*ntree[1:], **opts)
except HyTypeError as e:
if e.expression is None:
e.expression = tree
@ -225,4 +225,4 @@ def tag_macroexpand(tag, tree, compiler):
)
expr = tag_macro(tree)
return replace_hy_obj(wrap_value(expr), tree)
return replace_hy_obj(expr, tree)

View File

@ -4,6 +4,7 @@
from __future__ import unicode_literals
from hy._compat import PY3, str_type, bytes_type, long_type, string_types
from fractions import Fraction
class HyObject(object):
@ -229,6 +230,8 @@ class HyExpression(HyList):
return "(%s)" % (" ".join([repr(x) for x in self]))
_wrappers[HyExpression] = lambda e: HyExpression(wrap_value(x) for x in e)
_wrappers[Fraction] = lambda e: HyExpression(
[HySymbol("fraction"), wrap_value(e.numerator), wrap_value(e.denominator)])
class HySet(HyList):

View File

@ -7,9 +7,9 @@ from __future__ import unicode_literals
from hy import HyString
from hy.models import HyObject
from hy.compiler import hy_compile
from hy.importer import import_buffer_to_hst
from hy.errors import HyCompileError, HyTypeError
from hy.lex.exceptions import LexException
from hy.lex import tokenize
from hy._compat import PY3
import ast
@ -25,12 +25,12 @@ def _ast_spotcheck(arg, root, secondary):
def can_compile(expr):
return hy_compile(tokenize(expr), "__main__")
return hy_compile(import_buffer_to_hst(expr), "__main__")
def cant_compile(expr):
try:
hy_compile(tokenize(expr), "__main__")
hy_compile(import_buffer_to_hst(expr), "__main__")
assert False
except HyTypeError as e:
# Anything that can't be compiled should raise a user friendly
@ -48,8 +48,10 @@ def cant_compile(expr):
def test_ast_bad_type():
"Make sure AST breakage can happen"
class C:
pass
try:
hy_compile("foo", "__main__")
hy_compile(C(), "__main__")
assert True is False
except HyCompileError:
pass
@ -252,7 +254,7 @@ def test_ast_require():
def test_ast_no_pointless_imports():
def contains_import_from(code):
return any([isinstance(node, ast.ImportFrom)
for node in hy_compile(tokenize(code), "__main__").body])
for node in can_compile(code).body])
# `reduce` is a builtin in Python 2, but not Python 3.
# The version of `map` that returns an iterator is a builtin in
# Python 3, but not Python 2.
@ -460,7 +462,7 @@ def test_ast_unicode_strings():
hy_s.start_line = hy_s.end_line = 0
hy_s.start_column = hy_s.end_column = 0
code = hy_compile([hy_s], "__main__")
code = hy_compile(hy_s, "__main__")
# code == ast.Module(body=[ast.Expr(value=ast.Str(s=xxx))])
return code.body[0].value.s
@ -471,7 +473,7 @@ def test_ast_unicode_strings():
def test_ast_unicode_vs_bytes():
def f(x): return hy_compile(tokenize(x), "__main__").body[0].value.s
def f(x): return can_compile(x).body[0].value.s
assert f('"hello"') == u"hello"
assert type(f('"hello"')) is (str if PY3 else unicode) # noqa
assert f('b"hello"') == (eval('b"hello"') if PY3 else "hello")

View File

@ -5,6 +5,7 @@
(import [tests.resources [kwtest function-with-a-dash]]
[os.path [exists isdir isfile]]
[sys :as systest]
re
[operator [or_]]
[hy.errors [HyTypeError]]
pytest)
@ -1087,10 +1088,42 @@
(defn test-eval-failure []
"NATIVE: test eval failure modes"
; yo dawg
(try (eval '(eval)) (except [e HyTypeError]) (else (assert False)))
(try (eval '(eval "snafu")) (except [e HyTypeError]) (else (assert False)))
(try (eval '(eval)) (except [e TypeError]) (else (assert False)))
(defclass C)
(try (eval (C)) (except [e TypeError]) (else (assert False)))
(try (eval 'False []) (except [e HyTypeError]) (else (assert False)))
(try (eval 'False {} 1) (except [e HyTypeError]) (else (assert False))))
(try (eval 'False {} 1) (except [e TypeError]) (else (assert False))))
(defn test-eval-quasiquote []
; https://github.com/hylang/hy/issues/1174
(for [x [
None False True
5 5.1
1/2
5j 5.1j 2+1j 1.2+3.4j
"" b""
"apple bloom" b"apple bloom" "⚘" b"\x00"
:mykeyword
[] #{} {}
[1 2 3] #{1 2 3} {"a" 1 "b" 2}]]
(assert (= (eval `(identity ~x)) x))
(assert (= (eval x) x)))
; Tuples wrap to HyLists, not HyExpressions.
(assert (= (eval (,)) []))
(assert (= (eval (, 1 2 3)) [1 2 3]))
(assert (= (eval `(+ "a" ~(+ "b" "c"))) "abc"))
(setv l ["a" "b"])
(setv n 1)
(assert (= (eval `(get ~l ~n) "b")))
(setv d {"a" 1 "b" 2})
(setv k "b")
(assert (= (eval `(get ~d ~k)) 2)))
(defn test-import-syntax []
@ -1366,7 +1399,9 @@
(assert (= (disassemble '(do (leaky) (leaky) (macros)))
"Module(\n body=[\n Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),\n Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),\n Expr(value=Call(func=Name(id='macros'), args=[], keywords=[], starargs=None, kwargs=None))])")))
(assert (= (disassemble '(do (leaky) (leaky) (macros)) True)
"leaky()\nleaky()\nmacros()")))
"leaky()\nleaky()\nmacros()"))
(assert (= (re.sub r"[L() ]" "" (disassemble `(+ ~(+ 1 1) 40) True))
"2+40")))
(defn test-attribute-access []

View File

@ -84,6 +84,14 @@
"NATIVE: test macro calling a plain function"
(assert (= 3 (bar 1 2))))
(defn test-optional-and-apply-in-macro []
; https://github.com/hylang/hy/issues/1154
(defn f [&rest args]
(+ "f:" (repr args)))
(defmacro mac [&optional x]
`(apply f [~x]))
(assert (= (mac) "f:(None,)")))
(defn test-midtree-yield []
"NATIVE: test yielding with a returnable"
(defn kruft [] (yield) (+ 1 1)))