Merge pull request #1471 from vodik/async/await

Initial commit of asyncfn/await support

AUTHORS

@@ -86,3 +86,4 @@
 * Rob Day <rkd@rkd.me.uk>
 * Eric Kaschalk <ekaschalk@gmail.com>
 * Yoan Tournade <yoan@ytotech.com>
+* Simon Gomizelj <simon@vodik.xyz>

NEWS

@@ -25,6 +25,7 @@ Changes from 0.13.0
    * `xi` from `hy.extra.anaphoric` is now the `#%` tag macro
    * `#%` works on any expression and has a new `&kwargs` parameter `%**`
    * new `doc` macro and `#doc` tag macro
+   * support for PEP 492 with `fn/a`, `defn/a`, `with/a` and `for/a`
 
    [ Bug Fixes ]
    * Numeric literals are no longer parsed as symbols when followed by a dot
@@ -52,6 +53,7 @@ Changes from 0.13.0
    * `else` clauses in `for` and `while` are recognized more reliably
    * Argument destructuring no longer interferes with function docstrings.
    * Multiple expressions are now allowed in `try`
+   * `(yield-from)` is now a syntax error
 
    [ Misc. Improvements ]
    * `read`, `read_str`, and `eval` are exposed and documented as top-level

conftest.py

@@ -1,7 +1,7 @@
 import _pytest
 import hy
 import os
-from hy._compat import PY3, PY35
+from hy._compat import PY3, PY35, PY36
 
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
@@ -10,7 +10,8 @@ def pytest_collect_file(parent, path):
             and NATIVE_TESTS in path.dirname + os.sep
             and path.basename != "__init__.hy"
             and not ("py3_only" in path.basename and not PY3)
-            and not ("py35_only" in path.basename and not PY35)):
+            and not ("py35_only" in path.basename and not PY35)
+            and not ("py36_only" in path.basename and not PY36)):
         m = _pytest.python.pytest_pycollect_makemodule(path, parent)
         # Spoof the module name to avoid hitting an assertion in pytest.
         m.name = m.name[:-len(".hy")] + ".py"

docs/language/api.rst

@@ -729,6 +729,17 @@ Parameters may have the following keywords in front of them:
     Availability: Python 3.
 
 
+defn/a
+------
+
+``defn/a`` macro is a variant of ``defn`` that instead defines
+coroutines. It takes three parameters: the *name* of the function to
+define, a vector of *parameters*, and the *body* of the function:
+
+.. code-block:: clj
+
+    (defn/a name [params] body)
+
 defmain
 -------
 
@@ -1035,6 +1046,24 @@ not execute.
     loop finished
 
 
+for/a
+-----
+
+``for/a`` behaves like ``for`` but is used to call a function for each
+element generated by an asyncronous generator expression. The results
+of each call are discarded and the ``for/a`` expression returns
+``None`` instead.
+
+.. code-block:: clj
+
+    ;; assuming that (side-effect) is a function that takes a single parameter
+    (for/a [element (agen)] (side-effect element))
+
+    ;; for/a can have an optional else block
+    (for/a [element (agen)] (side-effect element)
+         (else (side-effect-2)))
+
+
 genexpr
 -------
 
@@ -1298,6 +1327,14 @@ This can be confirmed via Python's built-in ``help`` function::
     Multiplies input by three and returns result
     (END)
 
+fn/a
+----
+
+``fn/a`` is a variant of ``fn`` than defines an anonymous coroutine.
+The parameters are similar to ``defn/a``: the first parameter is
+vector of parameters and the rest is the body of the function. ``fn/a`` returns a
+new coroutine.
+
 last
 -----------
 
@@ -1868,6 +1905,26 @@ case it returns ``None``. So, the previous example could also be written
 
     (print (with [f (open "NEWS")] (.read f)))
 
+with/a
+------
+
+``with/a`` behaves like ``with``, but is used to wrap the execution of
+a block within a asynchronous context manager. The context manager can
+then set up the local system and tear it down in a controlled manner
+asynchronously.
+
+.. code-block:: clj
+
+    (with/a [arg (expr)] block)
+
+    (with/a [(expr)] block)
+
+    (with/a [arg (expr) (expr)] block)
+
+``with/a`` returns the value of its last form, unless it suppresses an exception
+(because the context manager's ``__aexit__`` method returned true), in which
+case it returns ``None``.
+
 with-decorator
 --------------
 

hy/_compat.py

@@ -22,6 +22,7 @@ import sys
 
 PY3 = sys.version_info[0] >= 3
 PY35 = sys.version_info >= (3, 5)
+PY36 = sys.version_info >= (3, 6)
 
 str_type     = str   if PY3 else unicode      # NOQA
 bytes_type   = bytes if PY3 else str          # NOQA

hy/compiler.py

@@ -76,6 +76,9 @@ def _is_hy_builtin(name, module_name):
 
 
 _compile_table = {}
+_decoratables = (ast.FunctionDef, ast.ClassDef)
+if PY35:
+    _decoratables += (ast.AsyncFunctionDef,)
 
 
 def ast_str(foobar):
@@ -250,6 +253,8 @@ class Result(object):
                 var.arg = new_name
             elif isinstance(var, ast.FunctionDef):
                 var.name = new_name
+            elif PY35 and isinstance(var, ast.AsyncFunctionDef):
+                var.name = new_name
             else:
                 raise TypeError("Don't know how to rename a %s!" % (
                     var.__class__.__name__))
@@ -1130,13 +1135,19 @@ class HyASTCompiler(object):
         return node(expr, names=names)
 
     @builds("yield")
-    @builds("yield_from", iff=PY3)
     @checkargs(max=1)
     def compile_yield_expression(self, expr):
         ret = Result(contains_yield=(not PY3))
         if len(expr) > 1:
             ret += self.compile(expr[1])
-        node = asty.Yield if expr[0] == "yield" else asty.YieldFrom
+        return ret + asty.Yield(expr, value=ret.force_expr)
+
+    @builds("yield_from", iff=PY3)
+    @builds("await", iff=PY35)
+    @checkargs(1)
+    def compile_yield_from_or_await_expression(self, expr):
+        ret = Result() + self.compile(expr[1])
+        node = asty.YieldFrom if expr[0] == "yield_from" else asty.Await
         return ret + node(expr, value=ret.force_expr)
 
     @builds("import")
@@ -1293,25 +1304,26 @@ class HyASTCompiler(object):
     def compile_decorate_expression(self, expr):
         expr.pop(0)  # with-decorator
         fn = self.compile(expr.pop())
-        if not fn.stmts or not isinstance(fn.stmts[-1], (ast.FunctionDef,
+        if not fn.stmts or not isinstance(fn.stmts[-1], _decoratables):
-                                                         ast.ClassDef)):
             raise HyTypeError(expr, "Decorated a non-function")
         decorators, ret, _ = self._compile_collect(expr)
         fn.stmts[-1].decorator_list = decorators + fn.stmts[-1].decorator_list
         return ret + fn
 
     @builds("with*")
+    @builds("with/a*", iff=PY35)
     @checkargs(min=2)
     def compile_with_expression(self, expr):
-        expr.pop(0)  # with*
+        root = expr.pop(0)
 
         args = expr.pop(0)
         if not isinstance(args, HyList):
             raise HyTypeError(expr,
-                              "with expects a list, received `{0}'".format(
+                              "{0} expects a list, received `{1}'".format(
-                                  type(args).__name__))
+                                  root, type(args).__name__))
         if len(args) not in (1, 2):
-            raise HyTypeError(expr, "with needs [arg (expr)] or [(expr)]")
+            raise HyTypeError(expr,
+                              "{0} needs [arg (expr)] or [(expr)]".format(root))
 
         thing = None
         if len(args) == 2:
@@ -1330,10 +1342,11 @@ class HyASTCompiler(object):
             expr, targets=[name], value=asty.Name(
                 expr, id=ast_str("None"), ctx=ast.Load()))
 
-        the_with = asty.With(expr,
+        node = asty.With if root == "with*" else asty.AsyncWith
-                             context_expr=ctx.force_expr,
+        the_with = node(expr,
-                             optional_vars=thing,
+                        context_expr=ctx.force_expr,
-                             body=body.stmts)
+                        optional_vars=thing,
+                        body=body.stmts)
 
         if PY3:
             the_with.items = [ast.withitem(context_expr=ctx.force_expr,
@@ -1819,16 +1832,16 @@ class HyASTCompiler(object):
         return result
 
     @builds("for*")
+    @builds("for/a*", iff=PY35)
     @checkargs(min=1)
     def compile_for_expression(self, expression):
-        expression.pop(0)  # for
+        root = expression.pop(0)
 
         args = expression.pop(0)
-
         if not isinstance(args, HyList):
             raise HyTypeError(expression,
-                              "`for` expects a list, received `{0}`".format(
+                              "`{0}` expects a list, received `{1}`".format(
-                                  type(args).__name__))
+                                  root, type(args).__name__))
 
         try:
             target_name, iterable = args
@@ -1853,11 +1866,12 @@ class HyASTCompiler(object):
         body = self._compile_branch(expression)
         body += body.expr_as_stmt()
 
-        ret += asty.For(expression,
+        node = asty.For if root == 'for*' else asty.AsyncFor
-                        target=target,
+        ret += node(expression,
-                        iter=ret.force_expr,
+                    target=target,
-                        body=body.stmts,
+                    iter=ret.force_expr,
-                        orelse=orel.stmts)
+                    body=body.stmts,
+                    orelse=orel.stmts)
 
         ret.contains_yield = body.contains_yield
 
@@ -1890,12 +1904,15 @@ class HyASTCompiler(object):
         return ret
 
     @builds("fn", "fn*")
+    @builds("fn/a", iff=PY35)
     # The starred version is for internal use (particularly, in the
     # definition of `defn`). It ensures that a FunctionDef is
     # produced rather than a Lambda.
     @checkargs(min=1)
     def compile_function_def(self, expression):
-        force_functiondef = expression.pop(0) == "fn*"
+        root = expression.pop(0)
+        force_functiondef = root in ("fn*", "fn/a")
+        asyncdef = root == "fn/a"
 
         arglist = expression.pop(0)
         docstring = None
@@ -1904,7 +1921,7 @@ class HyASTCompiler(object):
 
         if not isinstance(arglist, HyList):
             raise HyTypeError(expression,
-                              "First argument to `fn' must be a list")
+                              "First argument to `{}' must be a list".format(root))
 
         (ret, args, defaults, stararg,
          kwonlyargs, kwonlydefaults, kwargs) = self._parse_lambda_list(arglist)
@@ -1980,11 +1997,12 @@ class HyASTCompiler(object):
 
         name = self.get_anon_var()
 
-        ret += asty.FunctionDef(expression,
+        node = asty.AsyncFunctionDef if asyncdef else asty.FunctionDef
-                                name=name,
+        ret += node(expression,
-                                args=args,
+                    name=name,
-                                body=body.stmts,
+                    args=args,
-                                decorator_list=[])
+                    body=body.stmts,
+                    decorator_list=[])
 
         ast_name = asty.Name(expression, id=name, ctx=ast.Load())
 

hy/core/bootstrap.hy

@@ -70,6 +70,15 @@
     (macro-error name "defn takes a parameter list as second argument"))
   `(setv ~name (fn* ~lambda-list ~@body)))
 
+(defmacro defn/a [name lambda-list &rest body]
+  "Define `name` as a function with `lambda-list` signature and body `body`."
+  (import hy)
+  (if (not (= (type name) hy.HySymbol))
+    (macro-error name "defn/a takes a name as first argument"))
+  (if (not (isinstance lambda-list hy.HyList))
+    (macro-error name "defn/a takes a parameter list as second argument"))
+  `(setv ~name (fn/a ~lambda-list ~@body)))
+
 (defmacro if-python2 [python2-form python3-form]
   "If running on python2, execute python2-form, else, execute python3-form"
   (import sys)

hy/core/macros.hy

@@ -43,6 +43,19 @@ be associated in pairs."
                                                other-kvs))]))))
 
 
+(defn _with [node args body]
+  (if (not (empty? args))
+    (do
+     (if (>= (len args) 2)
+       (do
+        (setv p1 (.pop args 0)
+              p2 (.pop args 0)
+              primary [p1 p2])
+        `(~node [~@primary] ~(_with node args body)))
+       `(~node [~@args] ~@body)))
+    `(do ~@body)))
+
+
 (defmacro with [args &rest body]
   "Wrap execution of `body` within a context manager given as bracket `args`.
 
@@ -51,16 +64,18 @@ Shorthand for nested with* loops:
   (with* [x foo]
     (with* [y bar]
       baz))."
-  (if (not (empty? args))
+  (_with 'with* args body))
-    (do
+
-     (if (>= (len args) 2)
+
-       (do
+(defmacro with/a [args &rest body]
-        (setv p1 (.pop args 0)
+  "Wrap execution of `body` with/ain a context manager given as bracket `args`.
-              p2 (.pop args 0)
+
-              primary [p1 p2])
+Shorthand for nested with/a* loops:
-        `(with* [~@primary] (with ~args ~@body)))
+  (with/a [x foo y bar] baz) ->
-       `(with* [~@args] ~@body)))
+  (with/a* [x foo]
-    `(do ~@body)))
+    (with/a* [y bar]
+      baz))."
+  (_with 'with/a* args body))
 
 
 (defmacro cond [&rest branches]
@@ -93,11 +108,7 @@ used as the result."
      root)))
 
 
-(defmacro for [args &rest body]
+(defn _for [node args body]
-  "Build a for-loop with `args` as a [element coll] bracket pair and run `body`.
-
-Args may contain multiple pairs, in which case it executes a nested for-loop
-in order of the given pairs."
   (setv body (list body))
   (if (empty? body)
     (macro-error None "`for' requires a body to evaluate"))
@@ -109,10 +120,26 @@ in order of the given pairs."
     (odd? (len args)) (macro-error args "`for' requires an even number of args.")
     (empty? body)     (macro-error None "`for' requires a body to evaluate")
     (empty? args)     `(do ~@body ~@belse)
-    (= (len args) 2)  `(for* [~@args] (do ~@body) ~@belse)
+    (= (len args) 2)  `(~node [~@args] (do ~@body) ~@belse)
     (do
       (setv alist (cut args 0 None 2))
-      `(for* [(, ~@alist) (genexpr (, ~@alist) [~@args])] (do ~@body) ~@belse))))
+      `(~node [(, ~@alist) (genexpr (, ~@alist) [~@args])] (do ~@body) ~@belse))))
+
+
+(defmacro for [args &rest body]
+  "Build a for-loop with `args` as a [element coll] bracket pair and run `body`.
+
+Args may contain multiple pairs, in which case it executes a nested for-loop
+in order of the given pairs."
+  (_for 'for* args body))
+
+
+(defmacro for/a [args &rest body]
+  "Build a for/a-loop with `args` as a [element coll] bracket pair and run `body`.
+
+Args may contain multiple pairs, in which case it executes a nested for/a-loop
+in order of the given pairs."
+  (_for 'for/a* args body))
 
 
 (defmacro -> [head &rest rest]

tests/compilers/test_ast.py

@@ -14,6 +14,7 @@ from hy.lex.exceptions import LexException
 from hy._compat import PY3
 
 import ast
+import pytest
 
 
 def _ast_spotcheck(arg, root, secondary):
@@ -651,3 +652,15 @@ def test_compiler_macro_tag_try():
     # https://github.com/hylang/hy/issues/1350
     can_compile("(defmacro foo [] (try None (except [] None)) `())")
     can_compile("(deftag foo [] (try None (except [] None)) `())")
+
+
+@pytest.mark.skipif(not PY3, reason="Python 3 required")
+def test_ast_good_yield_from():
+    "Make sure AST can compile valid yield-from"
+    can_compile("(yield-from [1 2])")
+
+
+@pytest.mark.skipif(not PY3, reason="Python 3 required")
+def test_ast_bad_yield_from():
+    "Make sure AST can't compile invalid yield-from"
+    cant_compile("(yield-from)")

tests/native_tests/py35_only_tests.hy

@@ -5,6 +5,8 @@
 ;; Tests where the emitted code relies on Python ≥3.5.
 ;; conftest.py skips this file when running on Python <3.5.
 
+(import [asyncio [get-event-loop sleep]])
+
 
 (defn test-unpacking-pep448-1star []
   (setv l [1 2 3])
@@ -24,3 +26,78 @@
   (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
   (defn fun [&optional a b c d e f] [a b c d e f])
   (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))
+
+
+(defn run-coroutine [coro]
+  "Run a coroutine until its done in the default event loop."""
+  (.run_until_complete (get-event-loop) (coro)))
+
+
+(defn test-fn/a []
+  (assert (= (run-coroutine (fn/a [] (await (sleep 0)) [1 2 3]))
+             [1 2 3])))
+
+
+(defn test-defn/a []
+  (defn/a coro-test []
+    (await (sleep 0))
+    [1 2 3])
+  (assert (= (run-coroutine coro-test) [1 2 3])))
+
+
+(defn test-decorated-defn/a []
+  (defn decorator [func] (fn/a [] (/ (await (func)) 2)))
+
+  #@(decorator
+      (defn/a coro-test []
+        (await (sleep 0))
+        42))
+  (assert (= (run-coroutine coro-test) 21)))
+
+
+(defclass AsyncWithTest []
+  (defn --init-- [self val]
+    (setv self.val val)
+    None)
+
+  (defn/a --aenter-- [self]
+    self.val)
+
+  (defn/a --aexit-- [self tyle value traceback]
+    (setv self.val None)))
+
+
+(defn test-single-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t (AsyncWithTest 1)]
+        (assert (= t 1))))))
+
+(defn test-two-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)]
+        (assert (= t1 1))
+        (assert (= t2 2))))))
+
+(defn test-thrice-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))
+
+(defn test-quince-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)
+               _ (AsyncWithTest 4)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))

tests/native_tests/py36_only_tests.hy

@@ -0,0 +1,39 @@
+;; Copyright 2017 the authors.
+;; This file is part of Hy, which is free software licensed under the Expat
+;; license. See the LICENSE.
+
+;; Tests where the emitted code relies on Python ≥3.6.
+;; conftest.py skips this file when running on Python <3.6.
+
+(import [asyncio [get-event-loop sleep]])
+
+
+(defn run-coroutine [coro]
+  "Run a coroutine until its done in the default event loop."""
+  (.run_until_complete (get-event-loop) (coro)))
+
+
+(defn test-for/a []
+  (defn/a numbers []
+    (for [i [1 2]]
+      (yield i)))
+
+  (run-coroutine
+    (fn/a []
+      (setv x 0)
+      (for/a [a (numbers)]
+        (setv x (+ x a)))
+      (assert (= x 3)))))
+
+(defn test-for/a-else []
+  (defn/a numbers []
+    (for [i [1 2]]
+      (yield i)))
+
+  (run-coroutine
+    (fn/a []
+      (setv x 0)
+      (for/a [a (numbers)]
+        (setv x (+ x a))
+        (else (setv x (+ x 50))))
+      (assert (= x 53)))))