From 844256b99bb78bca7e454e4919aa70f5514f6097 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 3 Oct 2017 14:47:35 -0700 Subject: [PATCH 01/12] Make Asty use static rather than instance methods This ensures `asty.Pass is asty.Pass`. --- hy/compiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index fe47c9b..bc743e4 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -111,13 +111,13 @@ def builds_model(*model_types): # ast.Foo(..., lineno=x.lineno, col_offset=x.col_offset) class Asty(object): def __getattr__(self, name): - setattr(Asty, name, lambda self, x, **kwargs: getattr(ast, name)( + setattr(Asty, name, staticmethod(lambda x, **kwargs: getattr(ast, name)( lineno=getattr( x, 'start_line', getattr(x, 'lineno', None)), col_offset=getattr( x, 'start_column', getattr(x, 'col_offset', None)), - **kwargs)) - return getattr(self, name) + **kwargs))) + return getattr(Asty, name) asty = Asty() From 5ffbb4b0eb047ec487ca551da8e4502fd6a38aef Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 3 Oct 2017 14:50:05 -0700 Subject: [PATCH 02/12] Add Result.lineno and Result.col_offset --- hy/compiler.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hy/compiler.py b/hy/compiler.py index bc743e4..348f80f 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -180,6 +180,22 @@ class Result(object): self.__used_expr = False self._expr = value + @property + def lineno(self): + if self._expr is not None: + return self._expr.lineno + if self.stmts: + return self.stmts[-1].lineno + return None + + @property + def col_offset(self): + if self._expr is not None: + return self._expr.col_offset + if self.stmts: + return self.stmts[-1].col_offset + return None + def add_imports(self, mod, imports): """Autoimport `imports` from `mod`""" self.imports[mod].update(imports) From 7a40561db8dec676828fbd56afe7407b6f1ebac9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 29 May 2018 14:36:10 -0700 Subject: [PATCH 03/12] Add tagged model patterns --- hy/model_patterns.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hy/model_patterns.py b/hy/model_patterns.py index 36b91ae..1d30ccf 100644 --- a/hy/model_patterns.py +++ b/hy/model_patterns.py @@ -9,6 +9,7 @@ from funcparserlib.parser import ( some, skip, many, finished, a, Parser, NoParseError, State) from functools import reduce from itertools import repeat +from collections import namedtuple from operator import add from math import isinf @@ -74,3 +75,11 @@ def times(lo, hi, parser): end = e.state.max return result, State(s.pos, end) return f + +Tag = namedtuple('Tag', ['tag', 'value']) + +def tag(tag_name, parser): + """Matches the given parser and produces a named tuple `(Tag tag value)` + with `tag` set to the given tag name and `value` set to the parser's + value.""" + return parser >> (lambda x: Tag(tag_name, x)) From ba1dc55e96834b26f77895f8c9aa1031f84d9a72 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 12 Jun 2018 10:43:17 -0700 Subject: [PATCH 04/12] Implement `lfor`, `sfor`, `gfor`, `dfor` --- hy/compiler.py | 112 ++++++++++++++++++++++++++- tests/native_tests/comprehensions.hy | 106 +++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tests/native_tests/comprehensions.hy diff --git a/hy/compiler.py b/hy/compiler.py index 348f80f..82be63b 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -7,7 +7,7 @@ from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex, HyString, HyBytes, HySymbol, HyFloat, HyList, HySet, HyDict, HySequence, wrap_value) from hy.model_patterns import (FORM, SYM, STR, sym, brackets, whole, notpexpr, - dolike, pexpr, times) + dolike, pexpr, times, Tag, tag) from funcparserlib.parser import some, many, oneplus, maybe, NoParseError from hy.errors import HyCompileError, HyTypeError @@ -1055,6 +1055,116 @@ class HyASTCompiler(object): value=value.force_expr, generators=gen) + _loopers = many( + tag('setv', sym(":setv") + FORM + FORM) | + tag('if', sym(":if") + FORM) | + tag('do', sym(":do") + FORM) | + tag('afor', sym(":async") + FORM + FORM) | + tag('for', FORM + FORM)) + @special(["lfor", "sfor", "gfor"], [_loopers, FORM]) + @special(["dfor"], [_loopers, brackets(FORM, FORM)]) + def compile_new_comp(self, expr, root, parts, final): + node_class = dict( + lfor=asty.ListComp, + dfor=asty.DictComp, + sfor=asty.SetComp, + gfor=asty.GeneratorExp)[str(root)] + + # Compile the final value (and for dictionary comprehensions, the final + # key). + if node_class is asty.DictComp: + key, elt = map(self.compile, final) + else: + key = None + elt = self.compile(final) + + # Compile the parts. + parts = [ + Tag(p.tag, self.compile(p.value) if p.tag in ["if", "do"] else [ + self._storeize(p.value[0], self.compile(p.value[0])), + self.compile(p.value[1])]) + for p in parts] + + # Produce a result. + if (elt.stmts or (key is not None and key.stmts) or + any(p.tag == 'do' or (p.value[1].stmts if p.tag in ("for", "afor", "setv") else p.value.stmts) + for p in parts)): + # The desired comprehension can't be expressed as a + # real Python comprehension. We'll write it as a nested + # loop in a function instead. + def f(parts): + # This function is called recursively to construct + # the nested loop. + if not parts: + if node_class is asty.DictComp: + ret = key + elt + val = asty.Tuple( + key, ctx=ast.Load(), + elts=[key.force_expr, elt.force_expr]) + else: + ret = elt + val = elt.force_expr + return ret + asty.Expr( + elt, value=asty.Yield(elt, value=val)) + (tagname, v), parts = parts[0], parts[1:] + if tagname in ("for", "afor"): + node = asty.AsyncFor if tagname == "afor" else asty.For + return v[1] + node( + v[1], target=v[0], iter=v[1].force_expr, body=f(parts).stmts, + orelse=[]) + elif tagname == "setv": + return v[1] + asty.Assign( + v[1], targets=[v[0]], value=v[1].force_expr) + f(parts) + elif tagname == "if": + return v + asty.If( + v, test=v.force_expr, body=f(parts).stmts, orelse=[]) + elif tagname == "do": + return v + v.expr_as_stmt() + f(parts) + else: + raise ValueError("can't happen") + fname = self.get_anon_var() + # Define the generator function. + ret = Result() + asty.FunctionDef( + expr, + name=fname, + args=ast.arguments( + args=[], vararg=None, kwarg=None, + kwonlyargs=[], kw_defaults=[], defaults=[]), + body=f(parts).stmts, + decorator_list=[]) + # Immediately call the new function. Unless the user asked + # for a generator, wrap the call in `[].__class__(...)` or + # `{}.__class__(...)` or `{1}.__class__(...)` to get the + # right type. We don't want to just use e.g. `list(...)` + # because the name `list` might be rebound. + return ret + Result(expr=ast.parse( + "{}({}())".format( + {asty.ListComp: "[].__class__", + asty.DictComp: "{}.__class__", + asty.SetComp: "{1}.__class__", + asty.GeneratorExp: ""}[node_class], + fname)).body[0].value) + + # We can produce a real comprehension. + generators = [] + for tagname, v in parts: + if tagname in ("for", "afor"): + generators.append(ast.comprehension( + target=v[0], iter=v[1].expr, ifs=[], + is_async=int(tagname == "afor"))) + elif tagname == "setv": + generators.append(ast.comprehension( + target=v[0], + iter=asty.Tuple(v[1], elts=[v[1].expr], ctx=ast.Load()), + ifs=[], is_async=0)) + elif tagname == "if": + generators[-1].ifs.append(v.expr) + else: + raise ValueError("can't happen") + if node_class is asty.DictComp: + return asty.DictComp(expr, key=key.expr, value=elt.expr, generators=generators) + return node_class(expr, elt=elt.expr, generators=generators) + @special(["not", "~"], [FORM]) def compile_unary_operator(self, expr, root, arg): ops = {"not": ast.Not, diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy new file mode 100644 index 0000000..9c1a264 --- /dev/null +++ b/tests/native_tests/comprehensions.hy @@ -0,0 +1,106 @@ +(import + types + pytest) + + +(defn test-comprehension-types [] + + ; Forms that get compiled to real comprehensions + (assert (is (type (lfor x "abc" x)) list)) + (assert (is (type (sfor x "abc" x)) set)) + (assert (is (type (dfor x "abc" [x x])) dict)) + (assert (is (type (gfor x "abc" x)) types.GeneratorType)) + + ; Forms that get compiled to loops + (assert (is (type (lfor x "abc" :do (setv y 1) x)) list)) + (assert (is (type (sfor x "abc" :do (setv y 1) x)) set)) + (assert (is (type (dfor x "abc" :do (setv y 1) [x x])) dict)) + (assert (is (type (gfor x "abc" :do (setv y 1) x)) types.GeneratorType))) + + +#@ ((pytest.mark.parametrize "specialop" ["lfor" "sfor" "gfor" "dfor"]) +(defn test-comprehensions [specialop] + + (setv cases [ + ['(f x [] x) + []] + ['(f j [1 2 3] j) + [1 2 3]] + ['(f x (range 3) (* x 2)) + [0 2 4]] + ['(f x (range 2) y (range 2) (, x y)) + [(, 0 0) (, 0 1) (, 1 0) (, 1 1)]] + ['(f (, x y) (.items {"1" 1 "2" 2}) (* y 2)) + [2 4]] + ['(f x (do (setv s "x") "ab") y (do (+= s "y") "def") (+ x y s)) + ["adxy" "aexy" "afxy" "bdxyy" "bexyy" "bfxyy"]] + ['(f x (range 4) :if (% x 2) (* x 2)) + [2 6]] + ['(f x "abc" :setv y (.upper x) (+ x y)) + ["aA" "bB" "cC"]] + ['(f x "abc" :do (setv y (.upper x)) (+ x y)) + ["aA" "bB" "cC"]] + ['(f + x (range 3) + y (range 3) + :if (> y x) + z [7 8 9] + :setv s (+ x y z) + :if (!= z 8) + (, x y z s)) + [(, 0 1 7 8) (, 0 1 9 10) (, 0 2 7 9) (, 0 2 9 11) + (, 1 2 7 10) (, 1 2 9 12)]] + ['(f + x [0 1] + :setv l [] + y (range 4) + :do (.append l (, x y)) + :if (>= y 2) + z [7 8 9] + :if (!= z 8) + (, x y (tuple l) z)) + [(, 0 2 (, (, 0 0) (, 0 1) (, 0 2)) 7) + (, 0 2 (, (, 0 0) (, 0 1) (, 0 2)) 9) + (, 0 3 (, (, 0 0) (, 0 1) (, 0 2) (, 0 3)) 7) + (, 0 3 (, (, 0 0) (, 0 1) (, 0 2) (, 0 3)) 9) + (, 1 2 (, (, 1 0) (, 1 1) (, 1 2)) 7) + (, 1 2 (, (, 1 0) (, 1 1) (, 1 2)) 9) + (, 1 3 (, (, 1 0) (, 1 1) (, 1 2) (, 1 3)) 7) + (, 1 3 (, (, 1 0) (, 1 1) (, 1 2) (, 1 3)) 9)]] + + ['(f x (range 4) :do (unless (% x 2) (continue)) (* x 2)) + [2 6]] + ['(f x (range 4) :setv p 9 :do (unless (% x 2) (continue)) (* x 2)) + [2 6]] + ['(f x (range 20) :do (when (= x 3) (break)) (* x 2)) + [0 2 4]] + ['(f x (range 20) :setv p 9 :do (when (= x 3) (break)) (* x 2)) + [0 2 4]] + ['(f x [4 5] y (range 20) :do (when (> y 1) (break)) z [8 9] (, x y z)) + [(, 4 0 8) (, 4 0 9) (, 4 1 8) (, 4 1 9) + (, 5 0 8) (, 5 0 9) (, 5 1 8) (, 5 1 9)]]]) + + (for [[expr answer] cases] + ; Mutate the case as appropriate for the operator before + ; evaluating it. + (setv expr (+ (HyExpression [(HySymbol specialop)]) (cut expr 1))) + (when (= specialop "dfor") + (setv expr (+ (cut expr 0 -1) `([~(get expr -1) 1])))) + (setv result (eval expr)) + (when (= specialop "dfor") + (setv result (.keys result))) + (assert (= (sorted result) answer) (str expr))))) + + +(defn test-raise-in-comp [] + (defclass E [Exception] []) + (setv l []) + (import pytest) + (with [(pytest.raises E)] + (lfor + x (range 10) + :do (.append l x) + :do (when (= x 5) + (raise (E))) + x)) + (assert (= l [0 1 2 3 4 5]))) From 3256932b13afdc67e53a48c9699e5460edf90c5e Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 12 Jun 2018 10:54:08 -0700 Subject: [PATCH 05/12] Add a version of `for` parallel to `lfor` etc. --- hy/compiler.py | 54 ++++++++++---- hy/core/macros.hy | 9 +-- tests/compilers/test_ast.py | 4 +- tests/native_tests/comprehensions.hy | 87 +++++++++++++++++++++- tests/native_tests/contrib/walk.hy | 2 +- tests/native_tests/language.hy | 104 --------------------------- 6 files changed, 129 insertions(+), 131 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 82be63b..fb47f4b 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1061,24 +1061,39 @@ class HyASTCompiler(object): tag('do', sym(":do") + FORM) | tag('afor', sym(":async") + FORM + FORM) | tag('for', FORM + FORM)) + @special(["for"], [brackets(_loopers), + many(notpexpr("else")) + maybe(dolike("else"))]) @special(["lfor", "sfor", "gfor"], [_loopers, FORM]) @special(["dfor"], [_loopers, brackets(FORM, FORM)]) def compile_new_comp(self, expr, root, parts, final): - node_class = dict( - lfor=asty.ListComp, - dfor=asty.DictComp, - sfor=asty.SetComp, - gfor=asty.GeneratorExp)[str(root)] + root = unmangle(ast_str(root)) + node_class = { + "for": asty.For, + "lfor": asty.ListComp, + "dfor": asty.DictComp, + "sfor": asty.SetComp, + "gfor": asty.GeneratorExp}[root] + is_for = root == "for" - # Compile the final value (and for dictionary comprehensions, the final - # key). - if node_class is asty.DictComp: - key, elt = map(self.compile, final) + orel = [] + if is_for: + # Get the `else`. + body, else_expr = final + if else_expr is not None: + orel.append(self._compile_branch(else_expr)) + orel[0] += orel[0].expr_as_stmt() else: - key = None - elt = self.compile(final) + # Get the final value (and for dictionary + # comprehensions, the final key). + if node_class is asty.DictComp: + key, elt = map(self.compile, final) + else: + key = None + elt = self.compile(final) # Compile the parts. + if is_for: + parts = parts[0] parts = [ Tag(p.tag, self.compile(p.value) if p.tag in ["if", "do"] else [ self._storeize(p.value[0], self.compile(p.value[0])), @@ -1086,16 +1101,24 @@ class HyASTCompiler(object): for p in parts] # Produce a result. - if (elt.stmts or (key is not None and key.stmts) or + if (is_for or elt.stmts or (key is not None and key.stmts) or any(p.tag == 'do' or (p.value[1].stmts if p.tag in ("for", "afor", "setv") else p.value.stmts) for p in parts)): # The desired comprehension can't be expressed as a # real Python comprehension. We'll write it as a nested # loop in a function instead. + contains_yield = [] def f(parts): # This function is called recursively to construct # the nested loop. if not parts: + if is_for: + if body: + bd = self._compile_branch(body) + if bd.contains_yield: + contains_yield.append(True) + return bd + bd.expr_as_stmt() + return Result(stmts=[asty.Pass(expr)]) if node_class is asty.DictComp: ret = key + elt val = asty.Tuple( @@ -1108,10 +1131,11 @@ class HyASTCompiler(object): elt, value=asty.Yield(elt, value=val)) (tagname, v), parts = parts[0], parts[1:] if tagname in ("for", "afor"): + orelse = orel and orel.pop().stmts node = asty.AsyncFor if tagname == "afor" else asty.For return v[1] + node( v[1], target=v[0], iter=v[1].force_expr, body=f(parts).stmts, - orelse=[]) + orelse=orelse) elif tagname == "setv": return v[1] + asty.Assign( v[1], targets=[v[0]], value=v[1].force_expr) + f(parts) @@ -1122,6 +1146,10 @@ class HyASTCompiler(object): return v + v.expr_as_stmt() + f(parts) else: raise ValueError("can't happen") + if is_for: + ret = f(parts) + ret.contains_yield = bool(contains_yield) + return ret fname = self.get_anon_var() # Define the generator function. ret = Result() + asty.FunctionDef( diff --git a/hy/core/macros.hy b/hy/core/macros.hy index a948fd8..b87f110 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -122,14 +122,6 @@ used as the result." `(~node [(, ~@alist) (genexpr (, ~@alist) [~@args])] (do ~@body) ~@belse)))) -(defmacro for [args &rest body] - "Build a for-loop with `args` as a [element coll] bracket pair and run `body`. - -Args may contain multiple pairs, in which case it executes a nested for-loop -in order of the given pairs." - (_for 'for* args body)) - - (defmacro for/a [args &rest body] "Build a for/a-loop with `args` as a [element coll] bracket pair and run `body`. @@ -163,6 +155,7 @@ the second form, the second result is inserted into the third form, and so on." ~@(map build-form expressions) ~f)) + (defmacro ->> [head &rest args] "Thread `head` last through the `rest` of the forms. diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index d232649..ad41b01 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -531,9 +531,7 @@ def test_for_compile_error(): can_compile("(fn [] (for)))") assert excinfo.value.message == "Ran into a RPAREN where it wasn't expected." - with pytest.raises(HyTypeError) as excinfo: - can_compile("(fn [] (for [x] x))") - assert excinfo.value.message == "`for' requires an even number of args." + cant_compile("(fn [] (for [x] x))") def test_attribute_access(): diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index 9c1a264..b450e2f 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -18,8 +18,8 @@ (assert (is (type (gfor x "abc" :do (setv y 1) x)) types.GeneratorType))) -#@ ((pytest.mark.parametrize "specialop" ["lfor" "sfor" "gfor" "dfor"]) -(defn test-comprehensions [specialop] +#@ ((pytest.mark.parametrize "specialop" ["for" "lfor" "sfor" "gfor" "dfor"]) +(defn test-fors [specialop] (setv cases [ ['(f x [] x) @@ -86,6 +86,12 @@ (setv expr (+ (HyExpression [(HySymbol specialop)]) (cut expr 1))) (when (= specialop "dfor") (setv expr (+ (cut expr 0 -1) `([~(get expr -1) 1])))) + (when (= specialop "for") + (setv expr `(do + (setv out []) + (for [~@(cut expr 1 -1)] + (.append out ~(get expr -1))) + out))) (setv result (eval expr)) (when (= specialop "dfor") (setv result (.keys result))) @@ -104,3 +110,80 @@ (raise (E))) x)) (assert (= l [0 1 2 3 4 5]))) + + +(defn test-for-loop [] + "NATIVE: test for loops" + (setv count1 0 count2 0) + (for [x [1 2 3 4 5]] + (setv count1 (+ count1 x)) + (setv count2 (+ count2 x))) + (assert (= count1 15)) + (assert (= count2 15)) + (setv count 0) + (for [x [1 2 3 4 5] + y [1 2 3 4 5]] + (setv count (+ count x y)) + (else + (+= count 1))) + (assert (= count 151)) + + (setv count 0) + ; multiple statements in the else branch should work + (for [x [1 2 3 4 5] + y [1 2 3 4 5]] + (setv count (+ count x y)) + (else + (+= count 1) + (+= count 10))) + (assert (= count 161)) + + ; don't be fooled by constructs that look like else + (setv s "") + (setv else True) + (for [x "abcde"] + (+= s x) + [else (+= s "_")]) + (assert (= s "a_b_c_d_e_")) + + (setv s "") + (with [(pytest.raises TypeError)] + (for [x "abcde"] + (+= s x) + ("else" (+= s "z")))) + (assert (= s "az")) + + (assert (= (list ((fn [] (for [x [[1] [2 3]] y x] (yield y))))) + (list-comp y [x [[1] [2 3]] y x]))) + (assert (= (list ((fn [] (for [x [[1] [2 3]] y x z (range 5)] (yield z))))) + (list-comp z [x [[1] [2 3]] y x z (range 5)])))) + + +(defn test-nasty-for-nesting [] + "NATIVE: test nesting for loops harder" + ;; This test and feature is dedicated to @nedbat. + + ;; OK. This next test will ensure that we call the else branch exactly + ;; once. + (setv flag 0) + (for [x (range 2) + y (range 2)] + (+ 1 1) + (else (setv flag (+ flag 2)))) + (assert (= flag 2))) + + +(defn test-empty-for [] + + (setv l []) + (defn f [] + (for [x (range 3)] + (.append l "a") + (yield x))) + (for [x (f)]) + (assert (= l ["a" "a" "a"])) + + (setv l []) + (for [x (f)] + (else (.append l "z"))) + (assert (= l ["a" "a" "a" "z"]))) diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy index e2d7a2a..7facd59 100644 --- a/tests/native_tests/contrib/walk.hy +++ b/tests/native_tests/contrib/walk.hy @@ -46,7 +46,7 @@ (assert (= (macroexpand-all '(with [a 1])) '(with* [a 1] (do)))) (assert (= (macroexpand-all '(with [a 1 b 2 c 3] (for [d c] foo))) - '(with* [a 1] (with* [b 2] (with* [c 3] (do (for* [d c] (do foo)))))))) + '(with* [a 1] (with* [b 2] (with* [c 3] (do (for [d c] foo))))))) (assert (= (macroexpand-all '(with [a 1] '(with [b 2]) `(with [c 3] diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 1aa6bef..8dd978b 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -176,110 +176,6 @@ (with [(pytest.raises TypeError)] ("when" 1 2))) ; A macro -(defn test-for-loop [] - "NATIVE: test for loops" - (setv count1 0 count2 0) - (for [x [1 2 3 4 5]] - (setv count1 (+ count1 x)) - (setv count2 (+ count2 x))) - (assert (= count1 15)) - (assert (= count2 15)) - (setv count 0) - (for [x [1 2 3 4 5] - y [1 2 3 4 5]] - (setv count (+ count x y)) - (else - (+= count 1))) - (assert (= count 151)) - - (setv count 0) - ; multiple statements in the else branch should work - (for [x [1 2 3 4 5] - y [1 2 3 4 5]] - (setv count (+ count x y)) - (else - (+= count 1) - (+= count 10))) - (assert (= count 161)) - - ; don't be fooled by constructs that look like else - (setv s "") - (setv else True) - (for [x "abcde"] - (+= s x) - [else (+= s "_")]) - (assert (= s "a_b_c_d_e_")) - - (setv s "") - (setv else True) - (with [(pytest.raises TypeError)] - (for [x "abcde"] - (+= s x) - ("else" (+= s "z")))) - (assert (= s "az")) - - (assert (= (list ((fn [] (for [x [[1] [2 3]] y x] (yield y))))) - (list-comp y [x [[1] [2 3]] y x]))) - (assert (= (list ((fn [] (for [x [[1] [2 3]] y x z (range 5)] (yield z))))) - (list-comp z [x [[1] [2 3]] y x z (range 5)]))) - - (setv l []) - (defn f [] - (for [x [4 9 2]] - (.append l (* 10 x)) - (yield x))) - (for [_ (f)]) - (assert (= l [40 90 20]))) - - -(defn test-nasty-for-nesting [] - "NATIVE: test nesting for loops harder" - ;; This test and feature is dedicated to @nedbat. - - ;; let's ensure empty iterating is an implicit do - (setv t 0) - (for [] (setv t 1)) - (assert (= t 1)) - - ;; OK. This first test will ensure that the else is hooked up to the - ;; for when we break out of it. - (for [x (range 2) - y (range 2)] - (break) - (else (raise Exception))) - - ;; OK. This next test will ensure that the else is hooked up to the - ;; "inner" iteration - (for [x (range 2) - y (range 2)] - (if (= y 1) (break)) - (else (raise Exception))) - - ;; OK. This next test will ensure that the else is hooked up to the - ;; "outer" iteration - (for [x (range 2) - y (range 2)] - (if (= x 1) (break)) - (else (raise Exception))) - - ;; OK. This next test will ensure that we call the else branch exactly - ;; once. - (setv flag 0) - (for [x (range 2) - y (range 2)] - (+ 1 1) - (else (setv flag (+ flag 2)))) - (assert (= flag 2)) - - (setv l []) - (defn f [] - (for [x [4 9 2]] - (.append l (* 10 x)) - (yield x))) - (for [_ (f)]) - (assert (= l [40 90 20]))) - - (defn test-while-loop [] "NATIVE: test while loops?" (setv count 5) From e1972c535fac571fc6609d7f032f4e9e1c0d5ff0 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 12 Jun 2018 10:57:57 -0700 Subject: [PATCH 06/12] Remove `for/a`, `for*`, and `for/a*` --- hy/compiler.py | 30 ------------------------- hy/core/bootstrap.hy | 2 +- hy/core/language.hy | 12 +++++----- hy/core/macros.hy | 32 +++++---------------------- hy/core/shadow.hy | 2 +- tests/native_tests/language.hy | 4 ++-- tests/native_tests/native_macros.hy | 4 ++-- tests/native_tests/py36_only_tests.hy | 8 +++---- tests/native_tests/py3_only_tests.hy | 4 ++-- 9 files changed, 23 insertions(+), 75 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index fb47f4b..cbe0e30 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1442,36 +1442,6 @@ class HyASTCompiler(object): return result - @special(["for*", (PY35, "for/a*")], - [brackets(FORM, FORM), many(notpexpr("else")), maybe(dolike("else"))]) - def compile_for_expression(self, expr, root, args, body, else_expr): - target_name, iterable = args - target = self._storeize(target_name, self.compile(target_name)) - - ret = Result() - - orel = Result() - if else_expr is not None: - for else_body in else_expr: - orel += self.compile(else_body) - orel += orel.expr_as_stmt() - - ret += self.compile(iterable) - - body = self._compile_branch(body) - body += body.expr_as_stmt() - - node = asty.For if root == 'for*' else asty.AsyncFor - ret += node(expr, - target=target, - iter=ret.force_expr, - body=body.stmts or [asty.Pass(expr)], - orelse=orel.stmts) - - ret.contains_yield = body.contains_yield - - return ret - @special(["while"], [FORM, many(notpexpr("else")), maybe(dolike("else"))]) def compile_while_expression(self, expr, root, cond, body, else_expr): cond_compiled = self.compile(cond) diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy index 14539c3..cc576bc 100644 --- a/hy/core/bootstrap.hy +++ b/hy/core/bootstrap.hy @@ -18,7 +18,7 @@ (% "received a `%s' instead of a symbol for macro name" (. (type name) __name__))))) - (for* [kw '[&kwonly &kwargs]] + (for [kw '[&kwonly &kwargs]] (if* (in kw lambda-list) (raise (hy.errors.HyTypeError macro-name (% "macros cannot use %s" diff --git a/hy/core/language.hy b/hy/core/language.hy index 3481bf1..3f0dd39 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -41,7 +41,7 @@ fs (tuple rfs)) (fn [&rest args &kwargs kwargs] (setv res (first-f #* args #** kwargs)) - (for* [f fs] + (for [f fs] (setv res (f res))) res)))) @@ -79,7 +79,7 @@ If the second argument `codegen` is true, generate python code instead." (defn distinct [coll] "Return a generator from the original collection `coll` with no duplicates." (setv seen (set) citer (iter coll)) - (for* [val citer] + (for [val citer] (if (not-in val seen) (do (yield val) @@ -159,7 +159,7 @@ Return series of accumulated sums (or other binary function results)." (setv it (iter iterable) total (next it)) (yield total) - (for* [element it] + (for [element it] (setv total (func total element)) (yield total))) @@ -193,7 +193,7 @@ Return series of accumulated sums (or other binary function results)." (defn _flatten [coll result] (if (coll? coll) - (do (for* [b coll] + (do (for [b coll] (_flatten b result))) (.append result coll)) result) @@ -402,9 +402,9 @@ Raises ValueError for (not (pos? n))." (if (not (pos? n)) (raise (ValueError "n must be positive"))) (setv citer (iter coll) skip (dec n)) - (for* [val citer] + (for [val citer] (yield val) - (for* [_ (range skip)] + (for [_ (range skip)] (try (next citer) (except [StopIteration] diff --git a/hy/core/macros.hy b/hy/core/macros.hy index b87f110..05e67e9 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -101,42 +101,20 @@ used as the result." (setv root (check-branch branch)) (setv latest-branch root) - (for* [branch branches] + (for [branch branches] (setv cur-branch (check-branch branch)) (.append latest-branch cur-branch) (setv latest-branch cur-branch)) root))) -(defn _for [node args body] - (setv body (list body)) - (setv belse (if (and body (isinstance (get body -1) HyExpression) (= (get body -1 0) "else")) - [(body.pop)] - [])) - (if - (odd? (len args)) (macro-error args "`for' requires an even number of args.") - (empty? args) `(do ~@body ~@belse) - (= (len args) 2) `(~node [~@args] (do ~@body) ~@belse) - (do - (setv alist (cut args 0 None 2)) - `(~node [(, ~@alist) (genexpr (, ~@alist) [~@args])] (do ~@body) ~@belse)))) - - -(defmacro for/a [args &rest body] - "Build a for/a-loop with `args` as a [element coll] bracket pair and run `body`. - -Args may contain multiple pairs, in which case it executes a nested for/a-loop -in order of the given pairs." - (_for 'for/a* args body)) - - (defmacro -> [head &rest args] "Thread `head` first through the `rest` of the forms. The result of the first threaded form is inserted into the first position of the second form, the second result is inserted into the third form, and so on." (setv ret head) - (for* [node args] + (for [node args] (setv ret (if (isinstance node HyExpression) `(~(first node) ~ret ~@(rest node)) `(~node ~ret)))) @@ -162,7 +140,7 @@ the second form, the second result is inserted into the third form, and so on." The result of the first threaded form is inserted into the last position of the second form, the second result is inserted into the third form, and so on." (setv ret head) - (for* [node args] + (for [node args] (setv ret (if (isinstance node HyExpression) `(~@node ~ret) `(~node ~ret)))) @@ -203,7 +181,7 @@ the second form, the second result is inserted into the third form, and so on." (defmacro with-gensyms [args &rest body] "Execute `body` with `args` as bracket of names to gensym for use in macros." (setv syms []) - (for* [arg args] + (for [arg args] (.extend syms [arg `(gensym '~arg)])) `(do (setv ~@syms) @@ -218,7 +196,7 @@ the second form, the second result is inserted into the third form, and so on." (.startswith x "g!"))) (flatten body)))) gensyms []) - (for* [sym syms] + (for [sym syms] (.extend gensyms [sym `(gensym ~(cut sym 2))])) `(defmacro ~name [~@args] (setv ~@gensyms) diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy index c69f344..1f75108 100644 --- a/hy/core/shadow.hy +++ b/hy/core/shadow.hy @@ -161,7 +161,7 @@ (defn get [coll key1 &rest keys] "Access item in `coll` indexed by `key1`, with optional `keys` nested-access." (setv coll (get coll key1)) - (for* [k keys] + (for [k keys] (setv coll (get coll k))) coll) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 8dd978b..9ce8da2 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -830,13 +830,13 @@ (defn test-for-else [] "NATIVE: test for else" (setv x 0) - (for* [a [1 2]] + (for [a [1 2]] (setv x (+ x a)) (else (setv x (+ x 50)))) (assert (= x 53)) (setv x 0) - (for* [a [1 2]] + (for [a [1 2]] (setv x (+ x a)) (else)) (assert (= x 3))) diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index e4400ac..79ae560 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -101,7 +101,7 @@ (defn test-midtree-yield-in-for [] "NATIVE: test yielding in a for with a return" (defn kruft-in-for [] - (for* [i (range 5)] + (for [i (range 5)] (yield i)) (+ 1 2))) @@ -117,7 +117,7 @@ (defn test-multi-yield [] "NATIVE: testing multiple yields" (defn multi-yield [] - (for* [i (range 3)] + (for [i (range 3)] (yield i)) (yield "a") (yield "end")) diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy index 6a71fcf..d324a26 100644 --- a/tests/native_tests/py36_only_tests.hy +++ b/tests/native_tests/py36_only_tests.hy @@ -13,7 +13,7 @@ (.run_until_complete (get-event-loop) (coro))) -(defn test-for/a [] +(defn test-for-async [] (defn/a numbers [] (for [i [1 2]] (yield i))) @@ -21,11 +21,11 @@ (run-coroutine (fn/a [] (setv x 0) - (for/a [a (numbers)] + (for [:async a (numbers)] (setv x (+ x a))) (assert (= x 3))))) -(defn test-for/a-else [] +(defn test-for-async-else [] (defn/a numbers [] (for [i [1 2]] (yield i))) @@ -33,7 +33,7 @@ (run-coroutine (fn/a [] (setv x 0) - (for/a [a (numbers)] + (for [:async a (numbers)] (setv x (+ x a)) (else (setv x (+ x 50)))) (assert (= x 53))))) diff --git a/tests/native_tests/py3_only_tests.hy b/tests/native_tests/py3_only_tests.hy index 00cd448..1d69546 100644 --- a/tests/native_tests/py3_only_tests.hy +++ b/tests/native_tests/py3_only_tests.hy @@ -49,7 +49,7 @@ (defn test-yield-from [] "NATIVE: testing yield from" (defn yield-from-test [] - (for* [i (range 3)] + (for [i (range 3)] (yield i)) (yield-from [1 2 3])) (assert (= (list (yield-from-test)) [0 1 2 1 2 3]))) @@ -63,7 +63,7 @@ (yield 3) (assert 0)) (defn yield-from-test [] - (for* [i (range 3)] + (for [i (range 3)] (yield i)) (try (yield-from (yield-from-subgenerator-test)) From 4754b152a996549e209207ed3b46bb7861d3a47b Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 12 Jun 2018 11:00:01 -0700 Subject: [PATCH 07/12] Allow comprehensions with no looping parts --- hy/compiler.py | 7 +++++++ tests/native_tests/comprehensions.hy | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/hy/compiler.py b/hy/compiler.py index cbe0e30..d6bc53d 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1094,6 +1094,13 @@ class HyASTCompiler(object): # Compile the parts. if is_for: parts = parts[0] + if not parts: + return Result(expr=ast.parse({ + asty.For: "None", + asty.ListComp: "[]", + asty.DictComp: "{}", + asty.SetComp: "{1}.__class__()", + asty.GeneratorExp: "(_ for _ in [])"}[node_class]).body[0].value) parts = [ Tag(p.tag, self.compile(p.value) if p.tag in ["if", "do"] else [ self._storeize(p.value[0], self.compile(p.value[0])), diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index b450e2f..fcd0719 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -98,6 +98,18 @@ (assert (= (sorted result) answer) (str expr))))) +(defn test-fors-no-loopers [] + + (setv l []) + (for [] (.append l 1)) + (assert (= l [])) + + (assert (= (lfor 1) [])) + (assert (= (sfor 1) #{})) + (assert (= (list (gfor 1)) [])) + (assert (= (dfor [1 2]) {}))) + + (defn test-raise-in-comp [] (defclass E [Exception] []) (setv l []) From df4e49ec94f39a5ad9fc3e2d9e9e03272b686be6 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 7 Jun 2018 11:08:17 -0700 Subject: [PATCH 08/12] Test comprehension scoping --- tests/native_tests/comprehensions.hy | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index fcd0719..1f17912 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -1,6 +1,7 @@ (import types - pytest) + pytest + [hy._compat [PY3]]) (defn test-comprehension-types [] @@ -124,6 +125,30 @@ (assert (= l [0 1 2 3 4 5]))) +(defn test-scoping [] + + (setv x 0) + (for [x [1 2 3]]) + (assert (= x 3)) + + ; An `lfor` that gets compiled to a real comprehension + (setv x 0) + (assert (= (lfor x [1 2 3] (inc x)) [2 3 4])) + (assert (= x (if PY3 0 3))) + ; Python 2 list comprehensions leak their variables. + + ; An `lfor` that gets compiled to a loop + (setv x 0 l []) + (assert (= (lfor x [4 5 6] :do (.append l 1) (inc x)) [5 6 7])) + (assert (= l [1 1 1])) + (assert (= x 0)) + + ; An `sfor` that gets compiled to a real comprehension + (setv x 0) + (assert (= (sfor x [1 2 3] (inc x)) #{2 3 4})) + (assert (= x 0))) + + (defn test-for-loop [] "NATIVE: test for loops" (setv count1 0 count2 0) From cf0dafef9bed2fc61ba9ec3e94b5c6ac5ac20bba Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 7 Jun 2018 12:18:49 -0700 Subject: [PATCH 09/12] Update uses of the old comprehension forms --- hy/contrib/hy_repr.hy | 26 +++++++++++++++----------- hy/contrib/sequences.hy | 4 ++-- hy/core/language.hy | 6 +++--- hy/core/macros.hy | 10 +++++----- hy/core/shadow.hy | 3 +-- hy/extra/anaphoric.hy | 22 +++++++++++----------- tests/importer/test_importer.py | 2 +- tests/native_tests/comprehensions.hy | 4 ++-- tests/native_tests/contrib/hy_repr.hy | 6 +++--- tests/native_tests/tag_macros.hy | 2 +- tests/resources/pydemo.hy | 8 ++++---- 11 files changed, 48 insertions(+), 45 deletions(-) diff --git a/hy/contrib/hy_repr.hy b/hy/contrib/hy_repr.hy index 42ff62c..aca8b2a 100644 --- a/hy/contrib/hy_repr.hy +++ b/hy/contrib/hy_repr.hy @@ -25,9 +25,10 @@ (setv -seen (set)) (defn hy-repr [obj] (setv [f placeholder] (next - (genexpr (get -registry t) - [t (. (type obj) __mro__)] - (in t -registry)) + (gfor + t (. (type obj) __mro__) + :if (in t -registry) + (get -registry t)) [-base-repr None])) (global -quoting) @@ -55,18 +56,18 @@ ; collections.namedtuple.) (.format "({} {})" (. (type x) __name__) - (.join " " (genexpr (+ ":" k " " (hy-repr v)) [[k v] (zip x._fields x)]))) + (.join " " (gfor [k v] (zip x._fields x) (+ ":" k " " (hy-repr v))))) ; Otherwise, print it as a regular tuple. (+ "(," (if x " " "") (-cat x) ")")))) (hy-repr-register dict :placeholder "{...}" (fn [x] - (setv text (.join " " (genexpr - (+ (hy-repr k) " " (hy-repr v)) - [[k v] (.items x)]))) + (setv text (.join " " (gfor + [k v] (.items x) + (+ (hy-repr k) " " (hy-repr v))))) (+ "{" text "}"))) (hy-repr-register HyDict :placeholder "{...}" (fn [x] - (setv text (.join " " (genexpr - (+ (hy-repr k) " " (hy-repr v)) - [[k v] (partition x)]))) + (setv text (.join " " (gfor + [k v] (partition x) + (+ (hy-repr k) " " (hy-repr v))))) (if (% (len x) 2) (+= text (+ " " (hy-repr (get x -1))))) (+ "{" text "}"))) @@ -162,5 +163,8 @@ ; Call (.repr x) using the first class of x that doesn't inherit from ; HyObject. (.__repr__ - (next (genexpr t [t (. (type x) __mro__)] (not (issubclass t HyObject)))) + (next (gfor + t (. (type x) __mro__) + :if (not (issubclass t HyObject)) + t)) x)) diff --git a/hy/contrib/sequences.hy b/hy/contrib/sequences.hy index 703da2b..e684b80 100644 --- a/hy/contrib/sequences.hy +++ b/hy/contrib/sequences.hy @@ -11,8 +11,8 @@ --getitem-- (fn [self n] "get nth item of sequence" (if (hasattr n "start") - (genexpr (get self x) [x (range n.start n.stop - (or n.step 1))]) + (gfor x (range n.start n.stop (or n.step 1)) + (get self x)) (do (when (neg? n) ; Call (len) to force the whole ; sequence to be evaluated. diff --git a/hy/core/language.hy b/hy/core/language.hy index 3f0dd39..2895bb7 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -287,7 +287,7 @@ Return series of accumulated sums (or other binary function results)." "Return a function applying each `fs` to args, collecting results in a list." (setv fs (+ (, f) fs)) (fn [&rest args &kwargs kwargs] - (list-comp (f #* args #** kwargs) [f fs]))) + (lfor f fs (f #* args #** kwargs)))) (defn last [coll] "Return last item from `coll`." @@ -352,8 +352,8 @@ with overlap." (setv step (or step n) coll-clones (tee coll n) - slices (genexpr (islice (get coll-clones start) start None step) - [start (range n)])) + slices (gfor start (range n) + (islice (get coll-clones start) start None step))) (if (is fillvalue -sentinel) (zip #* slices) (zip-longest #* slices :fillvalue fillvalue))) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 05e67e9..3c65225 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -38,9 +38,9 @@ be associated in pairs." `(setv ~@(+ (if other-kvs [c coll] []) - #* (genexpr [`(get ~c ~k) v] - [[k v] (partition (+ (, k1 v1) - other-kvs))])))) + #* (gfor [k v] (partition (+ (, k1 v1) + other-kvs)) + [`(get ~c ~k) v])))) (defn _with [node args body] @@ -206,8 +206,8 @@ the second form, the second result is inserted into the third form, and so on." "Like `defmacro/g!`, with automatic once-only evaluation for 'o!' params. Such 'o!' params are available within `body` as the equivalent 'g!' symbol." - (setv os (list-comp s [s args] (.startswith s "o!")) - gs (list-comp (HySymbol (+ "g!" (cut s 2))) [s os])) + (setv os (lfor s args :if (.startswith s "o!") s) + gs (lfor s os (HySymbol (+ "g!" (cut s 2))))) `(defmacro/g! ~name ~args `(do (setv ~@(interleave ~gs ~os)) ~@~body))) diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy index 1f75108..ecf42a5 100644 --- a/hy/core/shadow.hy +++ b/hy/core/shadow.hy @@ -98,8 +98,7 @@ (defn comp-op [op a1 a-rest] "Helper for shadow comparison operators" (if a-rest - (reduce (fn [x y] (and x y)) - (list-comp (op x y) [(, x y) (zip (+ (, a1) a-rest) a-rest)])) + (and #* (gfor (, x y) (zip (+ (, a1) a-rest) a-rest) (op x y))) True)) (defn < [a1 &rest a-rest] "Shadowed `<` operator perform lt comparison on `a1` by each `a-rest`." diff --git a/hy/extra/anaphoric.hy b/hy/extra/anaphoric.hy index 25c187e..276f72d 100644 --- a/hy/extra/anaphoric.hy +++ b/hy/extra/anaphoric.hy @@ -109,18 +109,18 @@ `%*` and `%**` name the `&rest` and `&kwargs` parameters, respectively. Nesting of `#%` forms is not recommended." - (setv %symbols (set-comp a - [a (flatten [expr])] - (and (symbol? a) - (.startswith a '%)))) + (setv %symbols (sfor a (flatten [expr]) + :if (and (symbol? a) + (.startswith a '%)) + a)) `(fn [;; generate all %i symbols up to the maximum found in expr - ~@(genexpr (HySymbol (+ "%" (str i))) - [i (range 1 (-> (list-comp (int (cut a 1)) - [a %symbols] - (.isdigit (cut a 1))) - (or (, 0)) - max - inc))]) + ~@(gfor i (range 1 (-> (lfor a %symbols + :if (.isdigit (cut a 1)) + (int (cut a 1))) + (or (, 0)) + max + inc)) + (HySymbol (+ "%" (str i)))) ;; generate the &rest parameter only if '%* is present in expr ~@(if (in '%* %symbols) '(&rest %*)) diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 44669eb..8aca7c9 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -84,5 +84,5 @@ def test_eval(): assert eval_str('(.strip " fooooo ")') == 'fooooo' assert eval_str( '(if True "this is if true" "this is if false")') == "this is if true" - assert eval_str('(list-comp (pow num 2) [num (range 100)] (= (% num 2) 1))') == [ + assert eval_str('(lfor num (range 100) :if (= (% num 2) 1) (pow num 2))') == [ pow(num, 2) for num in range(100) if num % 2 == 1] diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy index 1f17912..9d97cbd 100644 --- a/tests/native_tests/comprehensions.hy +++ b/tests/native_tests/comprehensions.hy @@ -191,9 +191,9 @@ (assert (= s "az")) (assert (= (list ((fn [] (for [x [[1] [2 3]] y x] (yield y))))) - (list-comp y [x [[1] [2 3]] y x]))) + (lfor x [[1] [2 3]] y x y))) (assert (= (list ((fn [] (for [x [[1] [2 3]] y x z (range 5)] (yield z))))) - (list-comp z [x [[1] [2 3]] y x z (range 5)])))) + (lfor x [[1] [2 3]] y x z (range 5) z)))) (defn test-nasty-for-nesting [] diff --git a/tests/native_tests/contrib/hy_repr.hy b/tests/native_tests/contrib/hy_repr.hy index cdea7c4..2944d85 100644 --- a/tests/native_tests/contrib/hy_repr.hy +++ b/tests/native_tests/contrib/hy_repr.hy @@ -144,11 +144,11 @@ (setv x {1 2 3 [4 5] 6 7}) (setv (get x 3 1) x) - (assert (in (hy-repr x) (list-comp + (assert (in (hy-repr x) (lfor ; The ordering of a dictionary isn't guaranteed, so we need ; to check for all possible orderings. - (+ "{" (.join " " p) "}") - [p (permutations ["1 2" "3 [4 {...}]" "6 7"])])))) + p (permutations ["1 2" "3 [4 {...}]" "6 7"]) + (+ "{" (.join " " p) "}"))))) (defn test-matchobject [] (import re) diff --git a/tests/native_tests/tag_macros.hy b/tests/native_tests/tag_macros.hy index 5efedd1..4fb3659 100644 --- a/tests/native_tests/tag_macros.hy +++ b/tests/native_tests/tag_macros.hy @@ -110,7 +110,7 @@ ((wraps func) (fn [&rest args &kwargs kwargs] (func #* (map inc args) - #** (dict-comp k (inc v) [[k v] (.items kwargs)]))))) + #** (dfor [k v] (.items kwargs) [k (inc v)]))))) #@(increment-arguments (defn foo [&rest args &kwargs kwargs] diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy index 27d0ee4..b215375 100644 --- a/tests/resources/pydemo.hy +++ b/tests/resources/pydemo.hy @@ -29,10 +29,10 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little (setv myset #{4 5 6}) (setv mydict {7 8 9 900 10 15}) -(setv mylistcomp (list-comp x [x (range 10)] (% x 2))) -(setv mysetcomp (set-comp x [x (range 5)] (not (% x 2)))) -(setv mydictcomp (dict-comp k (.upper k) [k "abcde"] (!= k "c"))) -(setv mygenexpr (genexpr x [x (cycle [1 2 3])] (!= x 2))) +(setv mylistcomp (lfor x (range 10) :if (% x 2) x)) +(setv mysetcomp (sfor x (range 5) :if (not (% x 2)) x)) +(setv mydictcomp (dfor k "abcde" :if (!= k "c") [k (.upper k)])) +(setv mygenexpr (gfor x (cycle [1 2 3]) :if (!= x 2) x)) (setv attr-ref str.upper) (setv subscript (get "hello" 2)) From 14979edcab424d75005d93ce7d5811ba21abbd26 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 7 Jun 2018 12:20:23 -0700 Subject: [PATCH 10/12] Remove tests of the old comprehension forms --- tests/compilers/test_ast.py | 8 ------ tests/native_tests/language.hy | 46 ---------------------------------- 2 files changed, 54 deletions(-) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index ad41b01..bbf964a 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -554,14 +554,6 @@ def test_attribute_empty(): cant_compile('[2].foo') -def test_invalid_list_comprehension(): - """Ensure that invalid list comprehensions do not break the compiler""" - cant_compile("(genexpr x [])") - cant_compile("(genexpr [x [1 2 3 4]] x)") - cant_compile("(list-comp None [])") - cant_compile("(list-comp [x [1 2 3]] x)") - - def test_bad_setv(): """Ensure setv handles error cases""" cant_compile("(setv (a b) [1 2])") diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 9ce8da2..37f4d4d 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -842,52 +842,6 @@ (assert (= x 3))) -(defn test-list-comprehensions [] - "NATIVE: test list comprehensions" - (assert (= (list-comp (* x 2) [x (range 2)]) [0 2])) - (assert (= (list-comp (* x 2) [x (range 4)] (% x 2)) [2 6])) - (assert (= (sorted (list-comp (* y 2) [(, x y) (.items {"1" 1 "2" 2})])) - [2 4])) - (assert (= (list-comp (, x y) [x (range 2) y (range 2)]) - [(, 0 0) (, 0 1) (, 1 0) (, 1 1)])) - (assert (= (list-comp j [j [1 2]]) [1 2]))) - - -(defn test-set-comprehensions [] - "NATIVE: test set comprehensions" - (assert (instance? set (set-comp x [x (range 2)]))) - (assert (= (set-comp (* x 2) [x (range 2)]) (set [0 2]))) - (assert (= (set-comp (* x 2) [x (range 4)] (% x 2)) (set [2 6]))) - (assert (= (set-comp (* y 2) [(, x y) (.items {"1" 1 "2" 2})]) - (set [2 4]))) - (assert (= (set-comp (, x y) [x (range 2) y (range 2)]) - (set [(, 0 0) (, 0 1) (, 1 0) (, 1 1)]))) - (assert (= (set-comp j [j [1 2]]) (set [1 2])))) - - -(defn test-dict-comprehensions [] - "NATIVE: test dict comprehensions" - (assert (instance? dict (dict-comp x x [x (range 2)]))) - (assert (= (dict-comp x (* x 2) [x (range 2)]) {1 2 0 0})) - (assert (= (dict-comp x (* x 2) [x (range 4)] (% x 2)) {3 6 1 2})) - (assert (= (dict-comp x (* y 2) [(, x y) (.items {"1" 1 "2" 2})]) - {"2" 4 "1" 2})) - (assert (= (dict-comp (, x y) (+ x y) [x (range 2) y (range 2)]) - {(, 0 0) 0 (, 1 0) 1 (, 0 1) 1 (, 1 1) 2}))) - - -(defn test-generator-expressions [] - "NATIVE: test generator expressions" - (assert (not (instance? list (genexpr x [x (range 2)])))) - (assert (= (list (genexpr (* x 2) [x (range 2)])) [0 2])) - (assert (= (list (genexpr (* x 2) [x (range 4)] (% x 2))) [2 6])) - (assert (= (list (sorted (genexpr (* y 2) [(, x y) (.items {"1" 1 "2" 2})]))) - [2 4])) - (assert (= (list (genexpr (, x y) [x (range 2) y (range 2)])) - [(, 0 0) (, 0 1) (, 1 0) (, 1 1)])) - (assert (= (list (genexpr j [j [1 2]])) [1 2]))) - - (defn test-defn-order [] "NATIVE: test defn evaluation order" (setv acc []) From 76b80bad813a655ed1715d4f1203413257d757f2 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 7 Jun 2018 12:43:04 -0700 Subject: [PATCH 11/12] Remove support for the old comprehension forms --- hy/compiler.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index d6bc53d..2d065c2 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1020,41 +1020,6 @@ class HyASTCompiler(object): return gen_res + cond, gen - @special(["list-comp", "set-comp", "genexpr"], [FORM, FORM, maybe(FORM)]) - def compile_comprehension(self, expr, form, expression, gen, cond): - # (list-comp expr [target iter] cond?) - - if not isinstance(gen, HyList): - raise HyTypeError(gen, "Generator expression must be a list.") - - gen_res, gen = self._compile_generator_iterables( - [gen] + ([] if cond is None else [cond])) - - if len(gen) == 0: - raise HyTypeError(expr, "Generator expression cannot be empty.") - - ret = self.compile(expression) - node_class = ( - asty.ListComp if form == "list-comp" else - asty.SetComp if form == "set-comp" else - asty.GeneratorExp) - return ret + gen_res + node_class( - expr, elt=ret.force_expr, generators=gen) - - @special("dict-comp", [FORM, FORM, FORM, maybe(FORM)]) - def compile_dict_comprehension(self, expr, root, key, value, gen, cond): - key = self.compile(key) - value = self.compile(value) - - gen_res, gen = self._compile_generator_iterables( - [gen] + ([] if cond is None else [cond])) - - return key + value + gen_res + asty.DictComp( - expr, - key=key.force_expr, - value=value.force_expr, - generators=gen) - _loopers = many( tag('setv', sym(":setv") + FORM + FORM) | tag('if', sym(":if") + FORM) | @@ -1065,7 +1030,7 @@ class HyASTCompiler(object): many(notpexpr("else")) + maybe(dolike("else"))]) @special(["lfor", "sfor", "gfor"], [_loopers, FORM]) @special(["dfor"], [_loopers, brackets(FORM, FORM)]) - def compile_new_comp(self, expr, root, parts, final): + def compile_comprehension(self, expr, root, parts, final): root = unmangle(ast_str(root)) node_class = { "for": asty.For, From da754c0e5d4b7e6a2f34b0345c6c694b27319cea Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 12 Jun 2018 11:13:06 -0700 Subject: [PATCH 12/12] Update NEWS and docs for the new comprehensions --- NEWS.rst | 6 + docs/extra/reserved.rst | 2 +- docs/language/api.rst | 255 +++++++++++++++++++++------------------- docs/tutorial.rst | 17 ++- 4 files changed, 149 insertions(+), 131 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 35155f2..5ca0d90 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -14,6 +14,7 @@ Removals * Macros `ap-pipe` and `ap-compose` have been removed. Anaphoric macros do not work well with point-free style programming, in which case both threading macros and `comp` are more adequate. +* `for/a` has been removed. Use `(for [:async ...] ...)` instead. Other Breaking Changes ------------------------------ @@ -30,6 +31,10 @@ Other Breaking Changes * Non-shadow unary `=`, `is`, `<`, etc. now evaluate their argument instead of ignoring it. This change increases consistency a bit and makes accidental unary uses easier to notice. +* `list-comp`, `set-comp`, `dict-comp`, and `genexpr` have been replaced + by `lfor`, `sfor`, `dfor`, and `gfor`, respectively, which use a new + syntax and have additional features. All Python comprehensions can now + be written in Hy. * `hy-repr` uses registered functions instead of methods * `HyKeyword` no longer inherits from the string type and has been made into its own object type. @@ -47,6 +52,7 @@ New Features keyword arguments * Added a command-line option `-E` per CPython * `while` and `for` are allowed to have empty bodies +* `for` supports the various new clause types offered by `lfor` * Added a new module ``hy.model_patterns`` Bug Fixes diff --git a/docs/extra/reserved.rst b/docs/extra/reserved.rst index 26f853f..9ef4639 100644 --- a/docs/extra/reserved.rst +++ b/docs/extra/reserved.rst @@ -10,7 +10,7 @@ Usage: ``(names)`` This function can be used to get a list (actually, a ``frozenset``) of the names of Hy's built-in functions, macros, and special forms. The output also includes all Python reserved words. All names are in unmangled form -(e.g., ``list-comp`` rather than ``list_comp``). +(e.g., ``not-in`` rather than ``not_in``). .. code-block:: hy diff --git a/docs/language/api.rst b/docs/language/api.rst index 73066ee..68c9280 100644 --- a/docs/language/api.rst +++ b/docs/language/api.rst @@ -321,48 +321,17 @@ is only called on every other value in the list. (side-effect2 x)) -dict-comp ---------- - -``dict-comp`` is used to create dictionaries. It takes three or four parameters. -The first two parameters are for controlling the return value (key-value pair) -while the third is used to select items from a sequence. The fourth and optional -parameter can be used to filter out some of the items in the sequence based on a -conditional expression. - -.. code-block:: hy - - => (dict-comp x (* x 2) [x (range 10)] (odd? x)) - {1: 2, 3: 6, 9: 18, 5: 10, 7: 14} - - do ---------- -``do`` is used to evaluate each of its arguments and return the -last one. Return values from every other than the last argument are discarded. -It can be used in ``list-comp`` to perform more complex logic as shown in one -of the following examples. +``do`` (called ``progn`` in some Lisps) takes any number of forms, +evaluates them, and returns the value of the last one, or ``None`` if no +forms were provided. -Some example usage: +:: -.. code-block:: clj - - => (if True - ... (do (print "Side effects rock!") - ... (print "Yeah, really!"))) - Side effects rock! - Yeah, really! - - ;; assuming that (side-effect) is a function that we want to call for each - ;; and every value in the list, but whose return value we do not care about - => (list-comp (do (side-effect x) - ... (if (< x 5) (* 2 x) - ... (* 4 x))) - ... (x (range 10))) - [0, 2, 4, 6, 8, 20, 24, 28, 32, 36] - -``do`` can accept any number of arguments, from 1 to n. + => (+ 1 (do (setv x (+ 1 1)) x)) + 3 doc / #doc @@ -400,6 +369,20 @@ Gets help for macros or tag macros, respectively. Gets help for a tag macro function available in this module. +dfor +---- + +``dfor`` creates a :ref:`dictionary comprehension `. Its syntax +is the same as that of `lfor`_ except that the final value form must be +a literal list of two elements, the first of which becomes each key and +the second of which becomes each value. + +.. code-block:: hy + + => (dfor x (range 5) [x (* x 10)]) + {0: 0, 1: 10, 2: 20, 3: 30, 4: 40} + + setv ---- @@ -524,8 +507,8 @@ Parameters may have the following keywords in front of them: .. code-block:: clj => (defn zig-zag-sum [&rest numbers] - (setv odd-numbers (list-comp x [x numbers] (odd? x)) - even-numbers (list-comp x [x numbers] (even? x))) + (setv odd-numbers (lfor x numbers :if (odd? x) x) + even-numbers (lfor x numbers :if (even? x) x)) (- (sum odd-numbers) (sum even-numbers))) => (zig-zag-sum) @@ -850,24 +833,39 @@ raising an exception. for --- -``for`` is used to call a function for each element in a list or vector. -The results of each call are discarded and the ``for`` expression returns -``None`` instead. The example code iterates over *collection* and for each -*element* in *collection* calls the ``side-effect`` function with *element* -as its argument: +``for`` is used to evaluate some forms for each element in an iterable +object, such as a list. The return values of the forms are discarded and +the ``for`` form returns ``None``. -.. code-block:: clj +:: - ;; assuming that (side-effect) is a function that takes a single parameter - (for [element collection] (side-effect element)) + => (for [x [1 2 3]] + ... (print "iterating") + ... (print x)) + iterating + 1 + iterating + 2 + iterating + 3 - ;; for can have an optional else block - (for [element collection] (side-effect element) - (else (side-effect-2))) +In its square-bracketed first argument, ``for`` allows the same types of +clauses as lfor_. -The optional ``else`` block is only executed if the ``for`` loop terminates -normally. If the execution is halted with ``break``, the ``else`` block does -not execute. +:: + + => (for [x [1 2 3] :if (!= x 2) y [7 8]] + ... (print x y)) + 1 7 + 1 8 + 3 7 + 3 8 + +Furthermore, the last argument of ``for`` can be an ``(else …)`` form. +This form is executed after the last iteration of the ``for``\'s +outermost iteration clause, but only if that outermost loop terminates +normally. If it's jumped out of with e.g. ``break``, the ``else`` is +ignored. .. code-block:: clj @@ -888,43 +886,6 @@ not execute. loop finished -for/a ------ - -``for/a`` behaves like ``for`` but is used to call a function for each -element generated by an asynchronous generator expression. The results -of each call are discarded and the ``for/a`` expression returns -``None`` instead. - -.. code-block:: clj - - ;; assuming that (side-effect) is a function that takes a single parameter - (for/a [element (agen)] (side-effect element)) - - ;; for/a can have an optional else block - (for/a [element (agen)] (side-effect element) - (else (side-effect-2))) - - -genexpr -------- - -``genexpr`` is used to create generator expressions. It takes two or three -parameters. The first parameter is the expression controlling the return value, -while the second is used to select items from a list. The third and optional -parameter can be used to filter out some of the items in the list based on a -conditional expression. ``genexpr`` is similar to ``list-comp``, except it -returns an iterable that evaluates values one by one instead of evaluating them -immediately. - -.. code-block:: hy - - => (setv collection (range 10)) - => (setv filtered (genexpr x [x collection] (even? x))) - => (list filtered) - [0, 2, 4, 6, 8] - - .. _gensym: gensym @@ -977,6 +938,24 @@ successive elements in a nested structure. Example usage: index that is out of bounds. +gfor +---- + +``gfor`` creates a :ref:`generator expression `. Its syntax +is the same as that of `lfor`_. The difference is that ``gfor`` returns +an iterator, which evaluates and yields values one at a time. + +:: + + => (setv accum []) + => (list (take-while + ... (fn [x] (< x 5)) + ... (gfor x (count) :do (.append accum x) x))) + [0, 1, 2, 3, 4] + => accum + [0, 1, 2, 3, 4, 5] + + global ------ @@ -1190,27 +1169,69 @@ last 6 -list-comp ---------- +lfor +---- -``list-comp`` performs list comprehensions. It takes two or three parameters. -The first parameter is the expression controlling the return value, while -the second is used to select items from a list. The third and optional -parameter can be used to filter out some of the items in the list based on a -conditional expression. Some examples: +The comprehension forms ``lfor``, `sfor`_, `dfor`_, `gfor`_, and `for`_ +are used to produce various kinds of loops, including Python-style +:ref:`comprehensions `. ``lfor`` in particular +creates a list comprehension. A simple use of ``lfor`` is:: -.. code-block:: clj - - => (setv collection (range 10)) - => (list-comp x [x collection]) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - - => (list-comp (* x 2) [x collection]) - [0, 2, 4, 6, 8, 10, 12, 14, 16, 18] - - => (list-comp (* x 2) [x collection] (< x 5)) + => (lfor x (range 5) (* 2 x)) [0, 2, 4, 6, 8] +``x`` is the name of a new variable, which is bound to each element of +``(range 5)``. Each such element in turn is used to evaluate the value +form ``(* 2 x)``, and the results are accumulated into a list. + +Here's a more complex example:: + + => (lfor + ... x (range 3) + ... y (range 3) + ... :if (!= x y) + ... :setv total (+ x y) + ... [x y total]) + [[0, 1, 1], [0, 2, 2], [1, 0, 1], [1, 2, 3], [2, 0, 2], [2, 1, 3]] + +When there are several iteration clauses (here, the pairs of forms ``x +(range 3)`` and ``y (range 3)``), the result works like a nested loop or +Cartesian product: all combinations are considered in lexicographic +order. + +The general form of ``lfor`` is:: + + (lfor CLAUSES VALUE) + +where the ``VALUE`` is an arbitrary form that is evaluated to produce +each element of the result list, and ``CLAUSES`` is any number of +clauses. There are several types of clauses: + +- Iteration clauses, which look like ``LVALUE ITERABLE``. The ``LVALUE`` + is usually just a symbol, but could be something more complicated, + like ``[x y]``. +- ``:async LVALUE ITERABLE``, which is an + :ref:`asynchronous ` form of iteration clause. +- ``:do FORM``, which simply evaluates the ``FORM``. If you use + ``(continue)`` or ``(break)`` here, they will apply to the innermost + iteration clause before the ``:do``. +- ``:setv LVALUE RVALUE``, which is equivalent to ``:do (setv LVALUE + RVALUE)``. +- ``:if CONDITION``, which is equivalent to ``:do (unless CONDITION + (continue))``. + +For ``lfor``, ``sfor``, ``gfor``, and ``dfor``, variables are scoped as +if the comprehension form were its own function, so variables defined by +an iteration clause or ``:setv`` are not visible outside the form. In +fact, these forms are implemented as generator functions whenever they +contain Python statements, with the attendant consequences for calling +``return``. By contrast, ``for`` shares the caller's scope. + +.. note:: An exception to the above scoping rules occurs on Python 2 for + ``lfor`` specifically (and not ``sfor``, ``gfor``, or ``dfor``) when + Hy can implement the ``lfor`` as a Python list comprehension. Then, + variables will leak to the surrounding scope. + nonlocal -------- @@ -1465,20 +1486,12 @@ the end of a function, put ``None`` there yourself. => (print (f 4)) None -set-comp --------- -``set-comp`` is used to create sets. It takes two or three parameters. -The first parameter is for controlling the return value, while the second is -used to select items from a sequence. The third and optional parameter can be -used to filter out some of the items in the sequence based on a conditional -expression. +sfor +---- -.. code-block:: hy - - => (setv data [1 2 3 4 5 2 3 4 5 3 4 5]) - => (set-comp x [x data] (odd? x)) - {1, 3, 5} +``sfor`` creates a set comprehension. ``(sfor CLAUSES VALUE)`` is +equivalent to ``(set (lfor CLAUSES VALUE))``. See `lfor`_. cut @@ -1944,13 +1957,13 @@ infinite series without consuming infinite amount of memory. => (multiply (range 5) (range 5)) - => (list-comp value [value (multiply (range 10) (range 10))]) + => (list (multiply (range 10) (range 10))) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] => (import random) => (defn random-numbers [low high] ... (while True (yield (.randint random low high)))) - => (list-comp x [x (take 15 (random-numbers 1 50))]) + => (list (take 15 (random-numbers 1 50))) [7, 41, 6, 22, 32, 17, 5, 38, 18, 38, 17, 14, 23, 23, 19] diff --git a/docs/tutorial.rst b/docs/tutorial.rst index b85bf3a..70522e6 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -378,21 +378,20 @@ In Hy, you could do these like: .. code-block:: clj (setv odds-squared - (list-comp - (pow num 2) - (num (range 100)) - (= (% num 2) 1))) - + (lfor + num (range 100) + :if (= (% num 2) 1) + (pow num 2))) .. code-block:: clj ; And, an example stolen shamelessly from a Clojure page: ; Let's list all the blocks of a Chessboard: - (list-comp - (, x y) - (x (range 8) - y "ABCDEFGH")) + (lfor + x (range 8) + y "ABCDEFGH" + (, x y)) ; [(0, 'A'), (0, 'B'), (0, 'C'), (0, 'D'), (0, 'E'), (0, 'F'), (0, 'G'), (0, 'H'), ; (1, 'A'), (1, 'B'), (1, 'C'), (1, 'D'), (1, 'E'), (1, 'F'), (1, 'G'), (1, 'H'),