Implement #* and #** unpacking

This commit is contained in:
Kodi Arfer 2017-07-17 13:34:39 -07:00
parent c3c7af2db3
commit 2d863abc85
8 changed files with 155 additions and 21 deletions

View File

@ -1,12 +1,13 @@
import _pytest import _pytest
import hy import hy
from hy._compat import PY3 from hy._compat import PY3, PY35
def pytest_collect_file(parent, path): def pytest_collect_file(parent, path):
if (path.ext == ".hy" if (path.ext == ".hy"
and "/tests/native_tests/" in path.dirname + "/" and "/tests/native_tests/" in path.dirname + "/"
and path.basename != "__init__.hy" and path.basename != "__init__.hy"
and not ("py3_only" in path.basename and not PY3)): and not ("py3_only" in path.basename and not PY3)
and not ("py35_only" in path.basename and not PY35)):
m = _pytest.python.pytest_pycollect_makemodule(path, parent) m = _pytest.python.pytest_pycollect_makemodule(path, parent)
# Spoof the module name to avoid hitting an assertion in pytest. # Spoof the module name to avoid hitting an assertion in pytest.
m.name = m.name[:-len(".hy")] + ".py" m.name = m.name[:-len(".hy")] + ".py"

View File

@ -359,6 +359,13 @@ def checkargs(exact=None, min=None, max=None, even=None, multiple=None):
return _dec return _dec
def is_unpack(kind, x):
return (isinstance(x, HyExpression)
and len(x) > 0
and isinstance(x[0], HySymbol)
and x[0] == "unpack_" + kind)
class HyASTCompiler(object): class HyASTCompiler(object):
def __init__(self, module_name): def __init__(self, module_name):
@ -441,7 +448,8 @@ class HyASTCompiler(object):
raise HyCompileError(Exception("Unknown type: `%s'" % _type)) raise HyCompileError(Exception("Unknown type: `%s'" % _type))
def _compile_collect(self, exprs, with_kwargs=False): def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
oldpy_unpack=False):
"""Collect the expression contexts from a list of compiled expression. """Collect the expression contexts from a list of compiled expression.
This returns a list of the expression contexts, and the sum of the This returns a list of the expression contexts, and the sum of the
@ -451,10 +459,39 @@ class HyASTCompiler(object):
compiled_exprs = [] compiled_exprs = []
ret = Result() ret = Result()
keywords = [] keywords = []
oldpy_starargs = None
oldpy_kwargs = None
exprs_iter = iter(exprs) exprs_iter = iter(exprs)
for expr in exprs_iter: for expr in exprs_iter:
if with_kwargs and isinstance(expr, HyKeyword):
if not PY35 and oldpy_unpack and is_unpack("iterable", expr):
if oldpy_starargs:
raise HyTypeError(expr, "Pythons < 3.5 allow only one "
"`unpack-iterable` per call")
oldpy_starargs = self.compile(expr[1])
ret += oldpy_starargs
oldpy_starargs = oldpy_starargs.force_expr
elif is_unpack("mapping", expr):
ret += self.compile(expr[1])
if PY35:
if dict_display:
compiled_exprs.append(None)
compiled_exprs.append(ret.force_expr)
elif with_kwargs:
keywords.append(ast.keyword(
arg=None,
value=ret.force_expr,
lineno=expr.start_line,
col_offset=expr.start_column))
elif oldpy_unpack:
if oldpy_kwargs:
raise HyTypeError(expr, "Pythons < 3.5 allow only one "
"`unpack-mapping` per call")
oldpy_kwargs = ret.force_expr
elif with_kwargs and isinstance(expr, HyKeyword):
try: try:
value = next(exprs_iter) value = next(exprs_iter)
except StopIteration: except StopIteration:
@ -474,11 +511,15 @@ class HyASTCompiler(object):
value=compiled_value.force_expr, value=compiled_value.force_expr,
lineno=expr.start_line, lineno=expr.start_line,
col_offset=expr.start_column)) col_offset=expr.start_column))
else: else:
ret += self.compile(expr) ret += self.compile(expr)
compiled_exprs.append(ret.force_expr) compiled_exprs.append(ret.force_expr)
return compiled_exprs, ret, keywords if oldpy_unpack:
return compiled_exprs, ret, keywords, oldpy_starargs, oldpy_kwargs
else:
return compiled_exprs, ret, keywords
def _compile_branch(self, exprs): def _compile_branch(self, exprs):
return _branch(self.compile(expr) for expr in exprs) return _branch(self.compile(expr) for expr in exprs)
@ -610,6 +651,9 @@ class HyASTCompiler(object):
new_name = ast.Subscript(value=name.value, slice=name.slice) new_name = ast.Subscript(value=name.value, slice=name.slice)
elif isinstance(name, ast.Attribute): elif isinstance(name, ast.Attribute):
new_name = ast.Attribute(value=name.value, attr=name.attr) new_name = ast.Attribute(value=name.value, attr=name.attr)
elif PY3 and isinstance(name, ast.Starred):
new_name = ast.Starred(
value=self._storeize(expr, name.value, func))
else: else:
raise HyTypeError(expr, raise HyTypeError(expr,
"Can't assign or delete a %s" % "Can't assign or delete a %s" %
@ -717,6 +761,23 @@ class HyASTCompiler(object):
raise HyTypeError(expr, raise HyTypeError(expr,
"`%s' can't be used at the top-level" % expr[0]) "`%s' can't be used at the top-level" % expr[0])
@builds("unpack_iterable")
@checkargs(exact=1)
def compile_unpack_iterable(self, expr):
if not PY3:
raise HyTypeError(expr, "`unpack-iterable` isn't allowed here")
ret = self.compile(expr[1])
ret += ast.Starred(value=ret.force_expr,
lineno=expr.start_line,
col_offset=expr.start_column,
ctx=ast.Load())
return ret
@builds("unpack_mapping")
@checkargs(exact=1)
def compile_unpack_mapping(self, expr):
raise HyTypeError(expr, "`unpack-mapping` isn't allowed here")
@builds("do") @builds("do")
def compile_do(self, expression): def compile_do(self, expression):
expression.pop(0) expression.pop(0)
@ -2001,9 +2062,15 @@ class HyASTCompiler(object):
return self._compile_keyword_call(expression) return self._compile_keyword_call(expression)
if isinstance(fn, HySymbol): if isinstance(fn, HySymbol):
ret = self.compile_atom(fn, expression) # First check if `fn` is a special form, unless it has an
if ret: # `unpack_iterable` in it, since Python's operators (`+`,
return ret # etc.) can't unpack. An exception to this exception is that
# tuple literals (`,`) can unpack.
if fn == "," or not (
any(is_unpack("iterable", x) for x in expression[1:])):
ret = self.compile_atom(fn, expression)
if ret:
return ret
if fn.startswith("."): if fn.startswith("."):
# (.split "test test") -> "test test".split() # (.split "test test") -> "test test".split()
@ -2054,14 +2121,14 @@ class HyASTCompiler(object):
else: else:
with_kwargs = True with_kwargs = True
args, ret, kwargs = self._compile_collect(expression[1:], args, ret, keywords, oldpy_starargs, oldpy_kwargs = self._compile_collect(
with_kwargs) expression[1:], with_kwargs, oldpy_unpack=True)
ret += ast.Call(func=func.expr, ret += ast.Call(func=func.expr,
args=args, args=args,
keywords=kwargs, keywords=keywords,
starargs=None, starargs=oldpy_starargs,
kwargs=None, kwargs=oldpy_kwargs,
lineno=expression.start_line, lineno=expression.start_line,
col_offset=expression.start_column) col_offset=expression.start_column)
@ -2583,7 +2650,7 @@ class HyASTCompiler(object):
@builds(HyDict) @builds(HyDict)
def compile_dict(self, m): def compile_dict(self, m):
keyvalues, ret, _ = self._compile_collect(m) keyvalues, ret, _ = self._compile_collect(m, dict_display=True)
ret += ast.Dict(lineno=m.start_line, ret += ast.Dict(lineno=m.start_line,
col_offset=m.start_column, col_offset=m.start_column,

View File

@ -26,6 +26,7 @@ lg.add('QUASIQUOTE', r'`%s' % end_quote)
lg.add('UNQUOTESPLICE', r'~@%s' % end_quote) lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
lg.add('UNQUOTE', r'~%s' % end_quote) lg.add('UNQUOTE', r'~%s' % end_quote)
lg.add('HASHBANG', r'#!.*[^\r\n]') lg.add('HASHBANG', r'#!.*[^\r\n]')
lg.add('HASHSTARS', r'#\*+')
lg.add('HASHOTHER', r'#%s' % identifier) lg.add('HASHOTHER', r'#%s' % identifier)
# A regexp which matches incomplete strings, used to support # A regexp which matches incomplete strings, used to support

View File

@ -197,6 +197,22 @@ def term_unquote_splice(p):
return HyExpression([HySymbol("unquote_splice"), p[1]]) return HyExpression([HySymbol("unquote_splice"), p[1]])
@pg.production("term : HASHSTARS term")
@set_quote_boundaries
def term_hashstars(p):
n_stars = len(p[0].getstr()[1:])
if n_stars == 1:
sym = "unpack_iterable"
elif n_stars == 2:
sym = "unpack_mapping"
else:
raise LexException(
"Too many stars in `#*` construct (if you want to unpack a symbol "
"beginning with a star, separate it with whitespace)",
p[0].source_pos.lineno, p[0].source_pos.colno)
return HyExpression([HySymbol(sym), p[1]])
@pg.production("term : HASHOTHER term") @pg.production("term : HASHOTHER term")
@set_quote_boundaries @set_quote_boundaries
def hash_other(p): def hash_other(p):

View File

@ -369,6 +369,18 @@
(assert (is (isfile ".") False))) (assert (is (isfile ".") False)))
(defn test-star-unpacking []
; Python 3-only forms of unpacking are in py3_only_tests.hy
(setv l [1 2 3])
(setv d {"a" "x" "b" "y"})
(defn fun [&optional x1 x2 x3 x4 a b c] [x1 x2 x3 x4 a b c])
(assert (= (fun 5 #* l) [5 1 2 3 None None None]))
(assert (= (+ #* l) 6))
(assert (= (fun 5 #** d) [5 None None None "x" "y" None]))
(assert (= (fun 5 #* l #** d) [5 1 2 3 "x" "y" None])))
(defn test-kwargs [] (defn test-kwargs []
"NATIVE: test kwargs things." "NATIVE: test kwargs things."
(assert (= (apply kwtest [] {"one" "two"}) {"one" "two"})) (assert (= (apply kwtest [] {"one" "two"}) {"one" "two"}))

View File

@ -0,0 +1,26 @@
;; Copyright 2017 the authors.
;; This file is part of Hy, which is free software licensed under the Expat
;; license. See the LICENSE.
;; Tests where the emitted code relies on Python ≥3.5.
;; conftest.py skips this file when running on Python <3.5.
(defn test-unpacking-pep448-1star []
(setv l [1 2 3])
(setv p [4 5])
(assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3]))
(assert (= (, "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
(assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5}))
(defn f [&rest args] args)
(assert (= (f "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
(assert (= (+ #*l #*p) 15))
(assert (= (and #*l) 3)))
(defn test-unpacking-pep448-2star []
(setv d1 {"a" 1 "b" 2})
(setv d2 {"c" 3 "d" 4})
(assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
(defn fun [&optional a b c d e f] [a b c d e f])
(assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))

View File

@ -37,6 +37,16 @@
(, 1 2 (, 3 4) 5 {"bar" 6 "quux" 7})))) (, 1 2 (, 3 4) 5 {"bar" 6 "quux" 7}))))
(defn test-extended-unpacking-1star-lvalues []
(setv [x #*y] [1 2 3 4])
(assert (= x 1))
(assert (= y [2 3 4]))
(setv [a #*b c] "ghijklmno")
(assert (= a "g"))
(assert (= b (list "hijklmn")))
(assert (= c "o")))
(defn test-yield-from [] (defn test-yield-from []
"NATIVE: testing yield from" "NATIVE: testing yield from"
(defn yield-from-test [] (defn yield-from-test []

View File

@ -7,7 +7,7 @@
import os import os
import subprocess import subprocess
import re import re
from hy._compat import PY3 from hy._compat import PY3, PY35
from hy.importer import get_bytecode_path from hy.importer import get_bytecode_path
import pytest import pytest
@ -210,12 +210,13 @@ def test_hy2py():
if f.endswith(".hy"): if f.endswith(".hy"):
if f == "py3_only_tests.hy" and not PY3: if f == "py3_only_tests.hy" and not PY3:
continue continue
else: if f == "py35_only_tests.hy" and not PY35:
i += 1 continue
output, err = run_cmd("hy2py -s -a " + i += 1
os.path.join(dirpath, f)) output, err = run_cmd("hy2py -s -a " +
assert len(output) > 1, f os.path.join(dirpath, f))
assert len(err) == 0, f assert len(output) > 1, f
assert len(err) == 0, f
assert i assert i