Implement #* and #** unpacking

conftest.py

@@ -1,12 +1,13 @@
 import _pytest
 import hy
-from hy._compat import PY3
+from hy._compat import PY3, PY35
 
 def pytest_collect_file(parent, path):
     if (path.ext == ".hy"
             and "/tests/native_tests/" in path.dirname + "/"
             and path.basename != "__init__.hy"
-            and not ("py3_only" in path.basename and not PY3)):
+            and not ("py3_only" in path.basename and not PY3)
+            and not ("py35_only" in path.basename and not PY35)):
         m = _pytest.python.pytest_pycollect_makemodule(path, parent)
         # Spoof the module name to avoid hitting an assertion in pytest.
         m.name = m.name[:-len(".hy")] + ".py"

hy/compiler.py

@@ -359,6 +359,13 @@ def checkargs(exact=None, min=None, max=None, even=None, multiple=None):
     return _dec
 
 
+def is_unpack(kind, x):
+    return (isinstance(x, HyExpression)
+            and len(x) > 0
+            and isinstance(x[0], HySymbol)
+            and x[0] == "unpack_" + kind)
+
+
 class HyASTCompiler(object):
 
     def __init__(self, module_name):
@@ -441,7 +448,8 @@ class HyASTCompiler(object):
 
         raise HyCompileError(Exception("Unknown type: `%s'" % _type))
 
-    def _compile_collect(self, exprs, with_kwargs=False):
+    def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
+                         oldpy_unpack=False):
         """Collect the expression contexts from a list of compiled expression.
 
         This returns a list of the expression contexts, and the sum of the
@@ -451,10 +459,39 @@ class HyASTCompiler(object):
         compiled_exprs = []
         ret = Result()
         keywords = []
+        oldpy_starargs = None
+        oldpy_kwargs = None
 
         exprs_iter = iter(exprs)
         for expr in exprs_iter:
-            if with_kwargs and isinstance(expr, HyKeyword):
+
+            if not PY35 and oldpy_unpack and is_unpack("iterable", expr):
+                if oldpy_starargs:
+                    raise HyTypeError(expr, "Pythons < 3.5 allow only one "
+                                            "`unpack-iterable` per call")
+                oldpy_starargs = self.compile(expr[1])
+                ret += oldpy_starargs
+                oldpy_starargs = oldpy_starargs.force_expr
+
+            elif is_unpack("mapping", expr):
+                ret += self.compile(expr[1])
+                if PY35:
+                    if dict_display:
+                        compiled_exprs.append(None)
+                        compiled_exprs.append(ret.force_expr)
+                    elif with_kwargs:
+                        keywords.append(ast.keyword(
+                            arg=None,
+                            value=ret.force_expr,
+                            lineno=expr.start_line,
+                            col_offset=expr.start_column))
+                elif oldpy_unpack:
+                    if oldpy_kwargs:
+                        raise HyTypeError(expr, "Pythons < 3.5 allow only one "
+                                                "`unpack-mapping` per call")
+                    oldpy_kwargs = ret.force_expr
+
+            elif with_kwargs and isinstance(expr, HyKeyword):
                 try:
                     value = next(exprs_iter)
                 except StopIteration:
@@ -474,11 +511,15 @@ class HyASTCompiler(object):
                                             value=compiled_value.force_expr,
                                             lineno=expr.start_line,
                                             col_offset=expr.start_column))
+
             else:
                 ret += self.compile(expr)
                 compiled_exprs.append(ret.force_expr)
 
-        return compiled_exprs, ret, keywords
+        if oldpy_unpack:
+            return compiled_exprs, ret, keywords, oldpy_starargs, oldpy_kwargs
+        else:
+            return compiled_exprs, ret, keywords
 
     def _compile_branch(self, exprs):
         return _branch(self.compile(expr) for expr in exprs)
@@ -610,6 +651,9 @@ class HyASTCompiler(object):
             new_name = ast.Subscript(value=name.value, slice=name.slice)
         elif isinstance(name, ast.Attribute):
             new_name = ast.Attribute(value=name.value, attr=name.attr)
+        elif PY3 and isinstance(name, ast.Starred):
+            new_name = ast.Starred(
+                value=self._storeize(expr, name.value, func))
         else:
             raise HyTypeError(expr,
                               "Can't assign or delete a %s" %
@@ -717,6 +761,23 @@ class HyASTCompiler(object):
         raise HyTypeError(expr,
                           "`%s' can't be used at the top-level" % expr[0])
 
+    @builds("unpack_iterable")
+    @checkargs(exact=1)
+    def compile_unpack_iterable(self, expr):
+        if not PY3:
+            raise HyTypeError(expr, "`unpack-iterable` isn't allowed here")
+        ret = self.compile(expr[1])
+        ret += ast.Starred(value=ret.force_expr,
+                           lineno=expr.start_line,
+                           col_offset=expr.start_column,
+                           ctx=ast.Load())
+        return ret
+
+    @builds("unpack_mapping")
+    @checkargs(exact=1)
+    def compile_unpack_mapping(self, expr):
+        raise HyTypeError(expr, "`unpack-mapping` isn't allowed here")
+
     @builds("do")
     def compile_do(self, expression):
         expression.pop(0)
@@ -2001,9 +2062,15 @@ class HyASTCompiler(object):
             return self._compile_keyword_call(expression)
 
         if isinstance(fn, HySymbol):
-            ret = self.compile_atom(fn, expression)
-            if ret:
-                return ret
+            # First check if `fn` is a special form, unless it has an
+            # `unpack_iterable` in it, since Python's operators (`+`,
+            # etc.) can't unpack. An exception to this exception is that
+            # tuple literals (`,`) can unpack.
+            if fn == "," or not (
+                    any(is_unpack("iterable", x) for x in expression[1:])):
+                ret = self.compile_atom(fn, expression)
+                if ret:
+                    return ret
 
             if fn.startswith("."):
                 # (.split "test test") -> "test test".split()
@@ -2054,14 +2121,14 @@ class HyASTCompiler(object):
         else:
             with_kwargs = True
 
-        args, ret, kwargs = self._compile_collect(expression[1:],
-                                                  with_kwargs)
+        args, ret, keywords, oldpy_starargs, oldpy_kwargs = self._compile_collect(
+            expression[1:], with_kwargs, oldpy_unpack=True)
 
         ret += ast.Call(func=func.expr,
                         args=args,
-                        keywords=kwargs,
-                        starargs=None,
-                        kwargs=None,
+                        keywords=keywords,
+                        starargs=oldpy_starargs,
+                        kwargs=oldpy_kwargs,
                         lineno=expression.start_line,
                         col_offset=expression.start_column)
 
@@ -2583,7 +2650,7 @@ class HyASTCompiler(object):
 
     @builds(HyDict)
     def compile_dict(self, m):
-        keyvalues, ret, _ = self._compile_collect(m)
+        keyvalues, ret, _ = self._compile_collect(m, dict_display=True)
 
         ret += ast.Dict(lineno=m.start_line,
                         col_offset=m.start_column,

hy/lex/lexer.py

@@ -26,6 +26,7 @@ lg.add('QUASIQUOTE', r'`%s' % end_quote)
 lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
 lg.add('UNQUOTE', r'~%s' % end_quote)
 lg.add('HASHBANG', r'#!.*[^\r\n]')
+lg.add('HASHSTARS', r'#\*+')
 lg.add('HASHOTHER', r'#%s' % identifier)
 
 # A regexp which matches incomplete strings, used to support

hy/lex/parser.py

@@ -197,6 +197,22 @@ def term_unquote_splice(p):
     return HyExpression([HySymbol("unquote_splice"), p[1]])
 
 
+@pg.production("term : HASHSTARS term")
+@set_quote_boundaries
+def term_hashstars(p):
+    n_stars = len(p[0].getstr()[1:])
+    if n_stars == 1:
+        sym = "unpack_iterable"
+    elif n_stars == 2:
+        sym = "unpack_mapping"
+    else:
+        raise LexException(
+            "Too many stars in `#*` construct (if you want to unpack a symbol "
+            "beginning with a star, separate it with whitespace)",
+            p[0].source_pos.lineno, p[0].source_pos.colno)
+    return HyExpression([HySymbol(sym), p[1]])
+
+
 @pg.production("term : HASHOTHER term")
 @set_quote_boundaries
 def hash_other(p):

tests/native_tests/language.hy

@@ -369,6 +369,18 @@
   (assert (is (isfile ".") False)))
 
 
+(defn test-star-unpacking []
+  ; Python 3-only forms of unpacking are in py3_only_tests.hy
+  (setv l [1 2 3])
+  (setv d {"a" "x" "b" "y"})
+  (defn fun [&optional x1 x2 x3 x4 a b c] [x1 x2 x3 x4 a b c])
+  (assert (= (fun 5 #* l) [5 1 2 3 None None None]))
+  (assert (= (+ #* l) 6))
+  (assert (= (fun 5 #** d) [5 None None None "x" "y" None]))
+  (assert (= (fun 5 #* l #** d) [5 1 2 3 "x" "y" None])))
+
+
+
 (defn test-kwargs []
   "NATIVE: test kwargs things."
   (assert (= (apply kwtest [] {"one" "two"}) {"one" "two"}))

tests/native_tests/py35_only_tests.hy

@@ -0,0 +1,26 @@
+;; Copyright 2017 the authors.
+;; This file is part of Hy, which is free software licensed under the Expat
+;; license. See the LICENSE.
+
+;; Tests where the emitted code relies on Python ≥3.5.
+;; conftest.py skips this file when running on Python <3.5.
+
+
+(defn test-unpacking-pep448-1star []
+  (setv l [1 2 3])
+  (setv p [4 5])
+  (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3]))
+  (assert (= (, "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
+  (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5}))
+  (defn f [&rest args] args)
+  (assert (= (f "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
+  (assert (= (+ #*l #*p) 15))
+  (assert (= (and #*l) 3)))
+
+
+(defn test-unpacking-pep448-2star []
+  (setv d1 {"a" 1 "b" 2})
+  (setv d2 {"c" 3 "d" 4})
+  (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
+  (defn fun [&optional a b c d e f] [a b c d e f])
+  (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))

tests/native_tests/py3_only_tests.hy

@@ -37,6 +37,16 @@
              (, 1 2 (, 3 4)  5 {"bar" 6 "quux" 7}))))
 
 
+(defn test-extended-unpacking-1star-lvalues []
+  (setv [x #*y] [1 2 3 4])
+  (assert (= x 1))
+  (assert (= y [2 3 4]))
+  (setv [a #*b c] "ghijklmno")
+  (assert (= a "g"))
+  (assert (= b (list "hijklmn")))
+  (assert (= c "o")))
+
+
 (defn test-yield-from []
   "NATIVE: testing yield from"
   (defn yield-from-test []

tests/test_bin.py

@@ -7,7 +7,7 @@
 import os
 import subprocess
 import re
-from hy._compat import PY3
+from hy._compat import PY3, PY35
 from hy.importer import get_bytecode_path
 import pytest
 
@@ -210,12 +210,13 @@ def test_hy2py():
             if f.endswith(".hy"):
                 if f == "py3_only_tests.hy" and not PY3:
                     continue
-                else:
-                    i += 1
-                    output, err = run_cmd("hy2py -s -a " +
-                                          os.path.join(dirpath, f))
-                    assert len(output) > 1, f
-                    assert len(err) == 0, f
+                if f == "py35_only_tests.hy" and not PY35:
+                    continue
+                i += 1
+                output, err = run_cmd("hy2py -s -a " +
+                                      os.path.join(dirpath, f))
+                assert len(output) > 1, f
+                assert len(err) == 0, f
     assert i