Yaltik/hy
2
0
Fork 0
Code Pull Requests Releases Activity
51c7efe6e8
hy/tests/compilers/test_ast.py
Brandon T. Willard 51c7efe6e8 Retain compiled source and file information for exceptions 
This commit refactors the exception/error classes and their handling.
It also retains Hy source strings and their originating file information, when
available, all throughout the core parser and compiler functions.

As well, with these changes, calling code is no longer responsible for providing
source and file details to exceptions,

Closes hylang/hy#657.
2019-02-07 13:43:58 -05:00

690 lines
20 KiB
Python
Raw Blame History

# -*- encoding: utf-8 -*-
# Copyright 2019 the authors.
# This file is part of Hy, which is free software licensed under the Expat
# license. See the LICENSE.
from __future__ import unicode_literals
from hy import HyString
from hy.models import HyObject
from hy.compiler import hy_compile, hy_eval
from hy.errors import HyCompileError, HyTypeError
from hy.lex import hy_parse
from hy.lex.exceptions import LexException, PrematureEndOfInput
from hy._compat import PY3
import ast
import pytest
def _ast_spotcheck(arg, root, secondary):
if "." in arg:
local, full = arg.split(".", 1)
return _ast_spotcheck(full,
getattr(root, local),
getattr(secondary, local))
assert getattr(root, arg) == getattr(secondary, arg)
def can_compile(expr):
return hy_compile(hy_parse(expr), "__main__")
def can_eval(expr):
return hy_eval(hy_parse(expr))
def cant_compile(expr):
try:
hy_compile(hy_parse(expr), "__main__")
assert False
except HyTypeError as e:
# Anything that can't be compiled should raise a user friendly
# error, otherwise it's a compiler bug.
assert isinstance(e.expression, HyObject)
assert e.message
return e
except HyCompileError as e:
# Anything that can't be compiled should raise a user friendly
# error, otherwise it's a compiler bug.
assert isinstance(e.exception, HyTypeError)
assert e.traceback
return e
def s(x):
return can_compile('"module docstring" ' + x).body[-1].value.s
def test_ast_bad_type():
"Make sure AST breakage can happen"
class C:
pass
try:
hy_compile(C(), "__main__")
assert True is False
except TypeError:
pass
def test_empty_expr():
"Empty expressions should be illegal at the top level."
cant_compile("(print ())")
can_compile("(print '())")
def test_dot_unpacking():
can_compile("(.meth obj #* args az)")
cant_compile("(.meth #* args az)")
cant_compile("(. foo #* bar baz)")
can_compile("(.meth obj #** args az)")
can_compile("(.meth #** args obj)")
cant_compile("(. foo #** bar baz)")
def test_ast_bad_if():
"Make sure AST can't compile invalid if*"
cant_compile("(if*)")
cant_compile("(if* foobar)")
cant_compile("(if* 1 2 3 4 5)")
def test_ast_valid_if():
"Make sure AST can compile valid if*"
can_compile("(if* foo bar)")
def test_ast_valid_unary_op():
"Make sure AST can compile valid unary operator"
can_compile("(not 2)")
can_compile("(~ 1)")
def test_ast_invalid_unary_op():
"Make sure AST can't compile invalid unary operator"
cant_compile("(not 2 3 4)")
cant_compile("(not)")
cant_compile("(not 2 3 4)")
cant_compile("(~ 2 2 3 4)")
cant_compile("(~)")
def test_ast_bad_while():
"Make sure AST can't compile invalid while"
cant_compile("(while)")
def test_ast_good_do():
"Make sure AST can compile valid do"
can_compile("(do)")
can_compile("(do 1)")
def test_ast_good_raise():
"Make sure AST can compile valid raise"
can_compile("(raise)")
can_compile("(raise Exception)")
can_compile("(raise e)")
if PY3:
def test_ast_raise_from():
can_compile("(raise Exception :from NameError)")
def test_ast_bad_raise():
"Make sure AST can't compile invalid raise"
cant_compile("(raise Exception Exception)")
def test_ast_good_try():
"Make sure AST can compile valid try"
can_compile("(try 1 (except []) (else 1))")
can_compile("(try 1 (finally 1))")
can_compile("(try 1 (except []) (finally 1))")
can_compile("(try 1 (except [x]) (except [y]) (finally 1))")
can_compile("(try 1 (except []) (else 1) (finally 1))")
can_compile("(try 1 (except [x]) (except [y]) (else 1) (finally 1))")
def test_ast_bad_try():
"Make sure AST can't compile invalid try"
cant_compile("(try)")
cant_compile("(try 1)")
cant_compile("(try 1 bla)")
cant_compile("(try 1 bla bla)")
cant_compile("(try (do bla bla))")
cant_compile("(try (do) (else 1) (else 2))")
cant_compile("(try 1 (else 1))")
cant_compile("(try 1 (else 1) (except []))")
cant_compile("(try 1 (finally 1) (except []))")
cant_compile("(try 1 (except []) (finally 1) (else 1))")
def test_ast_good_except():
"Make sure AST can compile valid except"
can_compile("(try 1 (except []))")
can_compile("(try 1 (except [Foobar]))")
can_compile("(try 1 (except [[]]))")
can_compile("(try 1 (except [x FooBar]))")
can_compile("(try 1 (except [x [FooBar BarFoo]]))")
can_compile("(try 1 (except [x [FooBar BarFoo]]))")
def test_ast_bad_except():
"Make sure AST can't compile invalid except"
cant_compile("(except 1)")
cant_compile("(try 1 (except))")
cant_compile("(try 1 (except 1))")
cant_compile("(try 1 (except [1 3]))")
cant_compile("(try 1 (except [(f) [IOError ValueError]]))")
cant_compile("(try 1 (except [x [FooBar] BarBar]))")
def test_ast_good_assert():
"""Make sure AST can compile valid asserts. Asserts may or may not
include a label."""
can_compile("(assert 1)")
can_compile("(assert 1 \"Assert label\")")
can_compile("(assert 1 (+ \"spam \" \"eggs\"))")
can_compile("(assert 1 12345)")
can_compile("(assert 1 None)")
can_compile("(assert 1 (+ 2 \"incoming eggsception\"))")
def test_ast_bad_assert():
"Make sure AST can't compile invalid assert"
cant_compile("(assert)")
cant_compile("(assert 1 2 3)")
cant_compile("(assert 1 [1 2] 3)")
def test_ast_good_global():
"Make sure AST can compile valid global"
can_compile("(global a)")
can_compile("(global foo bar)")
def test_ast_bad_global():
"Make sure AST can't compile invalid global"
cant_compile("(global)")
cant_compile("(global (foo))")
if PY3:
def test_ast_good_nonlocal():
"Make sure AST can compile valid nonlocal"
can_compile("(nonlocal a)")
can_compile("(nonlocal foo bar)")
def test_ast_bad_nonlocal():
"Make sure AST can't compile invalid nonlocal"
cant_compile("(nonlocal)")
cant_compile("(nonlocal (foo))")
def test_ast_good_defclass():
"Make sure AST can compile valid defclass"
can_compile("(defclass a)")
can_compile("(defclass a [])")
can_compile("(defclass a [] None 42)")
can_compile("(defclass a [] None \"test\")")
can_compile("(defclass a [] None (print \"foo\"))")
@pytest.mark.skipif(not PY3, reason="Python 3 supports class keywords")
def test_ast_good_defclass_with_metaclass():
"Make sure AST can compile valid defclass with keywords"
can_compile("(defclass a [:metaclass b])")
can_compile("(defclass a [:b c])")
def test_ast_bad_defclass():
"Make sure AST can't compile invalid defclass"
cant_compile("(defclass)")
cant_compile("(defclass a None)")
cant_compile("(defclass a None None)")
def test_ast_good_lambda():
"Make sure AST can compile valid lambda"
can_compile("(fn [])")
can_compile("(fn [] 1)")
def test_ast_bad_lambda():
"Make sure AST can't compile invalid lambda"
cant_compile("(fn)")
cant_compile("(fn ())")
cant_compile("(fn () 1)")
cant_compile("(fn (x) 1)")
cant_compile('(fn "foo")')
def test_ast_good_yield():
"Make sure AST can compile valid yield"
can_compile("(yield 1)")
def test_ast_bad_yield():
"Make sure AST can't compile invalid yield"
cant_compile("(yield 1 2)")
def test_ast_import_mangle_dotted():
"""Mangling a module name with a period shouldn't create a spurious
`asname`."""
code = can_compile("(import a-b.c)")
assert code.body[0].names[0].name == "a_b.c"
assert code.body[0].names[0].asname is None
def test_ast_good_import_from():
"Make sure AST can compile valid selective import"
can_compile("(import [x [y]])")
def test_ast_require():
"Make sure AST respects (require) syntax"
can_compile("(require tests.resources.tlib)")
can_compile("(require [tests.resources.tlib [qplah parald]])")
can_compile("(require [tests.resources.tlib [*]])")
can_compile("(require [tests.resources.tlib :as foobar])")
can_compile("(require [tests.resources.tlib [qplah :as quiz]])")
can_compile("(require [tests.resources.tlib [qplah :as quiz parald]])")
cant_compile("(require [tests.resources.tlib])")
cant_compile("(require [tests.resources.tlib [* qplah]])")
cant_compile("(require [tests.resources.tlib [qplah *]])")
cant_compile("(require [tests.resources.tlib [* *]])")
def test_ast_import_require_dotted():
"""As in Python, it should be a compile-time error to attempt to
import a dotted name."""
cant_compile("(import [spam [foo.bar]])")
cant_compile("(require [spam [foo.bar]])")
def test_ast_no_pointless_imports():
def contains_import_from(code):
return any([isinstance(node, ast.ImportFrom)
for node in can_compile(code).body])
# `reduce` is a builtin in Python 2, but not Python 3.
# The version of `map` that returns an iterator is a builtin in
# Python 3, but not Python 2.
if PY3:
assert contains_import_from("reduce")
assert not contains_import_from("map")
else:
assert not contains_import_from("reduce")
assert contains_import_from("map")
def test_ast_good_get():
"Make sure AST can compile valid get"
can_compile("(get x y)")
def test_ast_bad_get():
"Make sure AST can't compile invalid get"
cant_compile("(get)")
cant_compile("(get 1)")
def test_ast_good_cut():
"Make sure AST can compile valid cut"
can_compile("(cut x)")
can_compile("(cut x y)")
can_compile("(cut x y z)")
can_compile("(cut x y z t)")
def test_ast_bad_cut():
"Make sure AST can't compile invalid cut"
cant_compile("(cut)")
cant_compile("(cut 1 2 3 4 5)")
def test_ast_good_take():
"Make sure AST can compile valid 'take'"
can_compile("(take 1 [2 3])")
def test_ast_good_drop():
"Make sure AST can compile valid 'drop'"
can_compile("(drop 1 [2 3])")
def test_ast_good_assoc():
"Make sure AST can compile valid assoc"
can_compile("(assoc x y z)")
def test_ast_bad_assoc():
"Make sure AST can't compile invalid assoc"
cant_compile("(assoc)")
cant_compile("(assoc 1)")
cant_compile("(assoc 1 2)")
cant_compile("(assoc 1 2 3 4)")
def test_ast_bad_with():
"Make sure AST can't compile invalid with"
cant_compile("(with*)")
cant_compile("(with* [])")
cant_compile("(with* [] (pass))")
def test_ast_valid_while():
"Make sure AST can't compile invalid while"
can_compile("(while foo bar)")
can_compile("(while foo bar (else baz))")
def test_ast_valid_for():
"Make sure AST can compile valid for"
can_compile("(for [a 2] (print a))")
def test_nullary_break_continue():
can_compile("(while 1 (break))")
cant_compile("(while 1 (break 1))")
can_compile("(while 1 (continue))")
cant_compile("(while 1 (continue 1))")
def test_ast_expression_basics():
""" Ensure basic AST expression conversion works. """
code = can_compile("(foo bar)").body[0]
tree = ast.Expr(value=ast.Call(
func=ast.Name(
id="foo",
ctx=ast.Load(),
),
args=[
ast.Name(id="bar", ctx=ast.Load())
],
keywords=[],
starargs=None,
kwargs=None,
))
_ast_spotcheck("value.func.id", code, tree)
def test_ast_anon_fns_basics():
""" Ensure anon fns work. """
code = can_compile("(fn [x] (* x x))").body[0].value
assert type(code) == ast.Lambda
code = can_compile("(fn [x] (print \"multiform\") (* x x))").body[0]
assert type(code) == ast.FunctionDef
can_compile("(fn [x])")
cant_compile("(fn)")
def test_ast_non_decoratable():
""" Ensure decorating garbage breaks """
cant_compile("(with-decorator (foo) (* x x))")
def test_ast_lambda_lists():
"""Ensure the compiler chokes on invalid lambda-lists"""
cant_compile('(fn [&optional [a b c]] a)')
cant_compile('(fn [&optional [1 2]] (list 1 2))')
def test_ast_print():
code = can_compile("(print \"foo\")").body[0]
assert type(code.value) == ast.Call
def test_ast_tuple():
""" Ensure tuples work. """
code = can_compile("(, 1 2 3)").body[0].value
assert type(code) == ast.Tuple
def test_lambda_list_keywords_rest():
""" Ensure we can compile functions with lambda list keywords."""
can_compile("(fn [x &rest xs] (print xs))")
cant_compile("(fn [x &rest xs &rest ys] (print xs))")
can_compile("(fn [&optional a &rest xs] (print xs))")
def test_lambda_list_keywords_kwargs():
""" Ensure we can compile functions with &kwargs."""
can_compile("(fn [x &kwargs kw] (list x kw))")
cant_compile("(fn [x &kwargs xs &kwargs ys] (list x xs ys))")
can_compile("(fn [&optional x &kwargs kw] (list x kw))")
def test_lambda_list_keywords_kwonly():
"""Ensure we can compile functions with &kwonly if we're on Python
3, or fail with an informative message on Python 2."""
kwonly_demo = "(fn [&kwonly a [b 2]] (print 1) (print a b))"
if PY3:
code = can_compile(kwonly_demo)
for i, kwonlyarg_name in enumerate(('a', 'b')):
assert kwonlyarg_name == code.body[0].args.kwonlyargs[i].arg
assert code.body[0].args.kw_defaults[0] is None
assert code.body[0].args.kw_defaults[1].n == 2
else:
exception = cant_compile(kwonly_demo)
assert isinstance(exception, HyTypeError)
message = exception.args[0]
assert message == "&kwonly parameters require Python 3"
def test_lambda_list_keywords_mixed():
""" Ensure we can mix them up."""
can_compile("(fn [x &rest xs &kwargs kw] (list x xs kw))")
cant_compile("(fn [x &rest xs &fasfkey {bar \"baz\"}])")
if PY3:
can_compile("(fn [x &rest xs &kwonly kwoxs &kwargs kwxs]"
" (list x xs kwxs kwoxs))")
def test_missing_keyword_argument_value():
"""Ensure the compiler chokes on missing keyword argument values."""
with pytest.raises(HyTypeError) as excinfo:
can_compile("((fn [x] x) :x)")
assert excinfo.value.message == "Keyword argument :x needs a value."
def test_ast_unicode_strings():
"""Ensure we handle unicode strings correctly"""
def _compile_string(s):
hy_s = HyString(s)
code = hy_compile([hy_s], "__main__")
# We put hy_s in a list so it isn't interpreted as a docstring.
# code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))])
return code.body[0].value.elts[0].s
assert _compile_string("test") == "test"
assert _compile_string("\u03b1\u03b2") == "\u03b1\u03b2"
assert _compile_string("\xc3\xa9") == "\xc3\xa9"
def test_ast_unicode_vs_bytes():
assert s('"hello"') == u"hello"
assert type(s('"hello"')) is (str if PY3 else unicode) # noqa
assert s('b"hello"') == (eval('b"hello"') if PY3 else "hello")
assert type(s('b"hello"')) is (bytes if PY3 else str)
assert s('b"\\xa0"') == (bytes([160]) if PY3 else chr(160))
def test_ast_bracket_string():
assert s(r'#[[empty delims]]') == 'empty delims'
assert s(r'#[my delim[fizzle]my delim]') == 'fizzle'
assert s(r'#[[]]') == ''
assert s(r'#[my delim[]my delim]') == ''
assert type(s('#[X[hello]X]')) is (str if PY3 else unicode) # noqa
assert s(r'#[X[raw\nstring]X]') == 'raw\\nstring'
assert s(r'#[foozle[aa foozli bb ]foozle]') == 'aa foozli bb '
assert s(r'#[([unbalanced](]') == 'unbalanced'
assert s(r'#[(1💯@)} {a![hello world](1💯@)} {a!]') == 'hello world'
assert (s(r'''#[X[
Remove the leading newline, please.
]X]''') == 'Remove the leading newline, please.\n')
assert (s(r'''#[X[
Only one leading newline should be removed.
]X]''') == '\n\nOnly one leading newline should be removed.\n')
def test_compile_error():
"""Ensure we get compile error in tricky cases"""
with pytest.raises(HyTypeError) as excinfo:
can_compile("(fn [] (in [1 2 3]))")
def test_for_compile_error():
"""Ensure we get compile error in tricky 'for' cases"""
with pytest.raises(PrematureEndOfInput) as excinfo:
can_compile("(fn [] (for)")
assert excinfo.value.message == "Premature end of input"
with pytest.raises(LexException) as excinfo:
can_compile("(fn [] (for)))")
assert excinfo.value.message == "Ran into a RPAREN where it wasn't expected."
cant_compile("(fn [] (for [x] x))")
def test_attribute_access():
"""Ensure attribute access compiles correctly"""
can_compile("(. foo bar baz)")
can_compile("(. foo [bar] baz)")
can_compile("(. foo bar [baz] [0] quux [frob])")
can_compile("(. foo bar [(+ 1 2 3 4)] quux [frob])")
cant_compile("(. foo bar :baz [0] quux [frob])")
cant_compile("(. foo bar baz (0) quux [frob])")
cant_compile("(. foo bar baz [0] quux {frob})")
def test_attribute_empty():
"""Ensure using dot notation with a non-expression is an error"""
cant_compile(".")
cant_compile("foo.")
cant_compile(".foo")
cant_compile('"bar".foo')
cant_compile('[2].foo')
def test_bad_setv():
"""Ensure setv handles error cases"""
cant_compile("(setv (a b) [1 2])")
def test_defn():
"""Ensure that defn works correctly in various corner cases"""
cant_compile("(defn \"hy\" [] 1)")
cant_compile("(defn :hy [] 1)")
can_compile("(defn &hy [] 1)")
cant_compile('(defn hy "foo")')
def test_setv_builtins():
"""Ensure that assigning to a builtin fails, unless in a class"""
cant_compile("(setv None 42)")
can_compile("(defclass A [] (defn get [self] 42))")
can_compile("""
(defclass A []
(defn get [self] 42)
(defclass B []
(defn get [self] 42))
(defn if* [self] 0))
""")
def test_top_level_unquote():
with pytest.raises(HyTypeError) as excinfo:
can_compile("(unquote)")
assert excinfo.value.message == "The special form 'unquote' is not allowed here"
with pytest.raises(HyTypeError) as excinfo:
can_compile("(unquote-splice)")
assert excinfo.value.message == "The special form 'unquote-splice' is not allowed here"
def test_lots_of_comment_lines():
# https://github.com/hylang/hy/issues/1313
can_compile(1000 * ";\n")
def test_exec_star():
code = can_compile('(exec* "print(5)")').body[0]
assert type(code) == (ast.Expr if PY3 else ast.Exec)
if not PY3:
assert code.body.s == "print(5)"
assert code.globals is None
assert code.locals is None
code = can_compile('(exec* "print(a)" {"a" 3})').body[0]
assert type(code) == (ast.Expr if PY3 else ast.Exec)
if not PY3:
assert code.body.s == "print(a)"
assert code.globals.keys[0].s == "a"
assert code.locals is None
code = can_compile('(exec* "print(a + b)" {"a" "x"} {"b" "y"})').body[0]
assert type(code) == (ast.Expr if PY3 else ast.Exec)
if not PY3:
assert code.body.s == "print(a + b)"
assert code.globals.keys[0].s == "a"
assert code.locals.keys[0].s == "b"
def test_compiler_macro_tag_try():
"""Check that try forms within defmacro/deftag are compiled correctly"""
# https://github.com/hylang/hy/issues/1350
can_compile("(defmacro foo [] (try None (except [] None)) `())")
can_compile("(deftag foo [] (try None (except [] None)) `())")
@pytest.mark.skipif(not PY3, reason="Python 3 required")
def test_ast_good_yield_from():
"Make sure AST can compile valid yield-from"
can_compile("(yield-from [1 2])")
@pytest.mark.skipif(not PY3, reason="Python 3 required")
def test_ast_bad_yield_from():
"Make sure AST can't compile invalid yield-from"
cant_compile("(yield-from)")
def test_eval_generator_with_return():
"""Ensure generators with a return statement works."""
can_eval("(fn [] (yield 1) (yield 2) (return))")
def test_futures_imports():
"""Make sure __future__ imports go first, especially when builtins are
automatically added (e.g. via use of a builtin name like `name`)."""
hy_ast = can_compile((
'(import [__future__ [print_function]])\n'
'(import sys)\n'
'(setv name [1 2])'
'(print (first name))'))
assert hy_ast.body[0].module == '__future__'
assert hy_ast.body[1].module == 'hy.core.language'
hy_ast = can_compile((
'(import sys)\n'
'(import [__future__ [print_function]])\n'
'(setv name [1 2])'
'(print (first name))'))
assert hy_ast.body[0].module == '__future__'
assert hy_ast.body[1].module == 'hy.core.language'
View Git Blame Copy Permalink