Merge pull request #1810 from refi64/ann

Implement PEP 3107 & 526 annotations

NEWS.rst

@@ -17,6 +17,7 @@ New Features
   inline Python code.
 * All augmented assignment operators (except `%=` and `^=`) now allow
   more than two arguments.
+* PEP 3107 and PEP 526 function and variable annotations are now supported.
 
 Other Breaking Changes
 ------------------------------

docs/language/api.rst

@@ -8,6 +8,48 @@ Hy features a number of special forms that are used to help generate
 correct Python AST. The following are "special" forms, which may have
 behavior that's slightly unexpected in some situations.
 
+^
+-
+
+The ``^`` symbol is used to denote annotations in three different contexts:
+
+- Standalone variable annotations.
+- Variable annotations in a setv call.
+- Function argument annotations.
+
+They implement `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_ and
+`PEP 3107 <https://www.python.org/dev/peps/pep-3107/>`_.
+
+Here is some example syntax of all three usages:
+
+.. code-block:: clj
+
+  ; Annotate the variable x as an int (equivalent to `x: int`).
+  (^int x)
+  ; Can annotate with expressions if needed (equivalent to `y: f(x)`).
+  (^(f x) y)
+
+  ; Annotations with an assignment: each annotation (int, str) covers the term that
+  ; immediately follows.
+  ; Equivalent to: x: int = 1; y = 2; z: str = 3
+  (setv ^int x 1 y 2 ^str z 3)
+
+  ; Annotate a as an int, c as an int, and b as a str.
+  ; Equivalent to: def func(a: int, b: str = None, c: int = 1): ...
+  (defn func [^int a &optional ^str b ^int [c 1]] ...)
+
+The rules are:
+
+- The value to annotate with is the value that immediately follows the caret.
+- There must be no space between the caret and the value to annotate, otherwise it will be
+  interpreted as a bitwise XOR like the Python operator.
+- The annotation always comes (and is evaluated) *before* the value being annotated. This is
+  unlike Python, where it comes and is evaluated *after* the value being annotated.
+
+Note that variable annotations are only supported on Python 3.6+.
+
+For annotating items with generic types, the of_ macro will likely be of use.
+
 .
 -
 
@@ -1408,6 +1450,33 @@ parameter will be returned.
 
 .. _py-specialform:
 
+of
+--
+
+``of`` is an alias for get, but with special semantics designed for handling PEP 484's generic
+types.
+
+``of`` has three forms:
+
+- ``(of T)`` will simply become ``T``.
+- ``(of T x)`` will become ``(get T x)``.
+- ``(of T x y ...)`` (where the ``...`` represents zero or more arguments) will become
+  ``(get T (, x y ...))``.
+
+For instance:
+
+.. code-block:: clj
+
+  (of str)  ; => str
+
+  (of List int)  ; => List[int]
+  (of Set int)   ; => Set[int]
+
+  (of Dict str str)   ; => Dict[str, str]
+  (of Tuple str int)  ; => Tuple[str, int]
+
+  (of Callable [int str] str)  ; => Callable[[int, str], str]
+
 py
 --
 

docs/tutorial.rst

@@ -287,7 +287,7 @@ If you run this program twice in a row, you'll see this::
     Executing
     Value: 1
     Done executing
-    $ hy example.hy 
+    $ hy example.hy
     Executing
     Value: 1
     Done executing

hy/compiler.py

@@ -23,6 +23,7 @@ import re
 import textwrap
 import pkgutil
 import traceback
+import itertools
 import importlib
 import inspect
 import types
@@ -337,6 +338,15 @@ def mklist(*items, **kwargs):
    return make_hy_model(HyList, items, kwargs.get('rest'))
 
 
+# Parse an annotation setting.
+OPTIONAL_ANNOTATION = maybe(pexpr(sym("annotate*") + FORM) >> (lambda x: x[0]))
+
+
+def is_annotate_expression(model):
+    return (isinstance(model, HyExpression) and model and isinstance(model[0], HySymbol)
+            and model[0] == HySymbol("annotate*"))
+
+
 class HyASTCompiler(object):
     """A Hy-to-Python AST compiler"""
 
@@ -1334,44 +1344,83 @@ class HyASTCompiler(object):
         return ret + asty.AugAssign(
             expr, target=target, value=ret.force_expr, op=op())
 
-    @special("setv", [many(FORM + FORM)])
+    @special("setv", [many(OPTIONAL_ANNOTATION + FORM + FORM)])
     @special((PY38, "setx"), [times(1, 1, SYM + FORM)])
-    def compile_def_expression(self, expr, root, pairs):
+    def compile_def_expression(self, expr, root, decls):
-        if not pairs:
+        if not decls:
             return asty.Name(expr, id='None', ctx=ast.Load())
+
         result = Result()
-        for pair in pairs:
+        is_assignment_expr = root == HySymbol("setx")
-            result += self._compile_assign(root, *pair)
+        for decl in decls:
+            if is_assignment_expr:
+                ann = None
+                name, value = decl
+            else:
+                ann, name, value = decl
+
+            result += self._compile_assign(ann, name, value,
+                                           is_assignment_expr=is_assignment_expr)
         return result
 
-    def _compile_assign(self, root, name, result):
+    @special(["annotate*"], [FORM, FORM])
+    def compile_basic_annotation(self, expr, root, ann, target):
+        return self._compile_assign(ann, target, None)
 
-        if name in [HySymbol(x) for x in ("None", "True", "False")]:
+    def _compile_assign(self, ann, name, value, *, is_assignment_expr = False):
-            raise self._syntax_error(name,
+        # Ensure that assignment expressions have a result and no annotation.
-                "Can't assign to `{}'".format(name))
+        assert not is_assignment_expr or (value is not None and ann is None)
 
-        result = self.compile(result)
         ld_name = self.compile(name)
 
-        if isinstance(ld_name.expr, ast.Call):
+        annotate_only = value is None
-            raise self._syntax_error(name,
+        if annotate_only:
-                "Can't assign to a callable: `{}'".format(name))
+            result = Result()
+        else:
+            result = self.compile(value)
+
+        invalid_name = False
+        if ann is not None:
+            # An annotation / annotated assignment is more strict with the target expression.
+            invalid_name = not isinstance(ld_name.expr, (ast.Name, ast.Attribute, ast.Subscript))
+        else:
+            invalid_name = (str(name) in ("None", "True", "False")
+                            or isinstance(ld_name.expr, ast.Call))
+
+        if invalid_name:
+            raise self._syntax_error(name, "illegal target for {}".format(
+                                        "annotation" if annotate_only else "assignment"))
 
         if (result.temp_variables
                 and isinstance(name, HySymbol)
                 and '.' not in name):
             result.rename(name)
-            if root != HySymbol("setx"):
+            if not is_assignment_expr:
                 # Throw away .expr to ensure that (setv ...) returns None.
                 result.expr = None
         else:
             st_name = self._storeize(name, ld_name)
-            node = (asty.NamedExpr
+
-                if root == HySymbol("setx")
+            if ann is not None:
-                else asty.Assign)
+                ann_result = self.compile(ann)
+                result = ann_result + result
+
+            if is_assignment_expr:
+                node = asty.NamedExpr
+            elif ann is not None:
+                if not PY36:
+                    raise self._syntax_error(name, "Variable annotations are not supported on "
+                                                   "Python <=3.6")
+
+                node = lambda x, **kw: asty.AnnAssign(x, annotation=ann_result.force_expr,
+                                                      simple=int(isinstance(name, HySymbol)),
+                                                      **kw)
+            else:
+                node = asty.Assign
+
             result += node(
                 name if hasattr(name, "start_line") else result,
-                value=result.force_expr,
+                value=result.force_expr if not annotate_only else None,
                 target=st_name, targets=[st_name])
 
         return result
@@ -1430,39 +1479,60 @@ class HyASTCompiler(object):
         # The starred version is for internal use (particularly, in the
         # definition of `defn`). It ensures that a FunctionDef is
         # produced rather than a Lambda.
+        OPTIONAL_ANNOTATION,
         brackets(
-            many(NASYM),
+            many(OPTIONAL_ANNOTATION + NASYM),
-            maybe(sym("&optional") + many(NASYM | brackets(SYM, FORM))),
+            maybe(sym("&optional") + many(OPTIONAL_ANNOTATION
-            maybe(sym("&rest") + NASYM),
+                                            + (NASYM | brackets(SYM, FORM)))),
-            maybe(sym("&kwonly") + many(NASYM | brackets(SYM, FORM))),
+            maybe(sym("&rest") + OPTIONAL_ANNOTATION + NASYM),
-            maybe(sym("&kwargs") + NASYM)),
+            maybe(sym("&kwonly") + many(OPTIONAL_ANNOTATION
+                                        + (NASYM | brackets(SYM, FORM)))),
+            maybe(sym("&kwargs") + OPTIONAL_ANNOTATION + NASYM)),
         many(FORM)])
-    def compile_function_def(self, expr, root, params, body):
+    def compile_function_def(self, expr, root, returns, params, body):
-
         force_functiondef = root in ("fn*", "fn/a")
         node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef
+        ret = Result()
+
+        # NOTE: Our evaluation order of return type annotations is
+        # different from Python: Python evalautes them after the argument
+        # annotations / defaults (as that's where they are in the source),
+        # but Hy evaluates them *first*, since here they come before the #
+        # argument list. Therefore, it would be more confusing for
+        # readability to evaluate them after like Python.
+
+        ret = Result()
+        returns_ann = None
+        if returns is not None:
+            returns_result = self.compile(returns)
+            ret += returns_result
 
         mandatory, optional, rest, kwonly, kwargs = params
-        optional, defaults, ret = self._parse_optional_args(optional)
-        kwonly, kw_defaults, ret2 = self._parse_optional_args(kwonly, True)
-        ret += ret2
-        main_args = mandatory + optional
 
-        main_args, kwonly, [rest], [kwargs] = (
+        optional = optional or []
-            [[x and asty.arg(x, arg=ast_str(x), annotation=None)
+        kwonly = kwonly or []
-              for x in o]
+
-             for o in (main_args or [], kwonly or [], [rest], [kwargs])])
+        mandatory_ast, _, ret = self._compile_arguments_set(mandatory, False, ret)
+        optional_ast, optional_defaults, ret = self._compile_arguments_set(optional, True, ret)
+        kwonly_ast, kwonly_defaults, ret = self._compile_arguments_set(kwonly, False, ret)
+
+        rest_ast = kwargs_ast = None
+
+        if rest is not None:
+            [rest_ast], _, ret = self._compile_arguments_set([rest], False, ret)
+        if kwargs is not None:
+            [kwargs_ast], _, ret = self._compile_arguments_set([kwargs], False, ret)
 
         args = ast.arguments(
-            args=main_args, defaults=defaults,
+            args=mandatory_ast + optional_ast, defaults=optional_defaults,
-            vararg=rest,
+            vararg=rest_ast,
             posonlyargs=[],
-            kwonlyargs=kwonly, kw_defaults=kw_defaults,
+            kwonlyargs=kwonly_ast, kw_defaults=kwonly_defaults,
-            kwarg=kwargs)
+            kwarg=kwargs_ast)
 
         body = self._compile_branch(body)
 
-        if not force_functiondef and not body.stmts:
+        if not force_functiondef and not body.stmts and returns is None:
             return ret + asty.Lambda(expr, args=args, body=body.force_expr)
 
         if body.expr:
@@ -1474,27 +1544,48 @@ class HyASTCompiler(object):
                     name=name,
                     args=args,
                     body=body.stmts or [asty.Pass(expr)],
-                    decorator_list=[])
+                    decorator_list=[],
+                    returns=returns_result.force_expr if returns is not None else None)
 
         ast_name = asty.Name(expr, id=name, ctx=ast.Load())
         ret += Result(expr=ast_name, temp_variables=[ast_name, ret.stmts[-1]])
         return ret
 
-    def _parse_optional_args(self, expr, allow_no_default=False):
+    def _compile_arguments_set(self, decls, implicit_default_none, ret):
-        # [a b [c 5] d] → ([a, b, c, d], [None, None, 5, d], <ret>)
+        args_ast = []
-        names, defaults, ret = [], [], Result()
+        args_defaults = []
-        for x in expr or []:
+
-            sym, value = (
+        for ann, decl in decls:
-                x if isinstance(x, HyList)
+            default = None
-                else (x, None) if allow_no_default
+
-                else (x, HySymbol('None').replace(x)))
+            # funcparserlib will check to make sure that the only times we
-            names.append(sym)
+            # ever have a HyList here are due to a default value.
-            if value is None:
+            if isinstance(decl, HyList):
-                defaults.append(None)
+                sym, default = decl
             else:
-                ret += self.compile(value)
+                sym = decl
-                defaults.append(ret.force_expr)
+                if implicit_default_none:
-        return names, defaults, ret
+                    default = HySymbol('None').replace(sym)
+
+            if ann is not None:
+                ret += self.compile(ann)
+                ann_ast = ret.force_expr
+            else:
+                ann_ast = None
+
+            if default is not None:
+                ret += self.compile(default)
+                args_defaults.append(ret.force_expr)
+            else:
+                # Note that the only time any None should ever appear here
+                # is in kwargs, since the order of those with defaults vs
+                # those without isn't significant in the same way as
+                # positional args.
+                args_defaults.append(None)
+
+            args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=ann_ast))
+
+        return args_ast, args_defaults, ret
 
     @special("return", [maybe(FORM)])
     def compile_return(self, expr, root, arg):
@@ -1543,12 +1634,22 @@ class HyASTCompiler(object):
             return expr
 
         new_args = []
-        pairs = list(expr[1:])
+        decls = list(expr[1:])
-        while pairs:
+        while decls:
-            k, v = (pairs.pop(0), pairs.pop(0))
+            if is_annotate_expression(decls[0]):
+                # Handle annotations.
+                ann = decls.pop(0)
+            else:
+                ann = None
+
+            k, v = (decls.pop(0), decls.pop(0))
             if ast_str(k) == "__init__" and isinstance(v, HyExpression):
                 v += HyExpression([HySymbol("None")])
-            new_args.extend([k, v])
+
+            if ann is not None:
+                new_args.append(ann)
+
+            new_args.extend((k, v))
         return HyExpression([HySymbol("setv")] + new_args).replace(expr)
 
     @special("dispatch-tag-macro", [STR, FORM])
@@ -1607,7 +1708,7 @@ class HyASTCompiler(object):
         return Result(stmts=o) if exec_mode else o
 
     @builds_model(HyExpression)
-    def compile_expression(self, expr):
+    def compile_expression(self, expr, *, allow_annotation_expression=False):
         # Perform macro expansions
         expr = macroexpand(expr, self.module, self)
         if not isinstance(expr, HyExpression):
@@ -1623,17 +1724,20 @@ class HyASTCompiler(object):
         func = None
 
         if isinstance(root, HySymbol):
-
             # First check if `root` is a special operator, unless it has an
             # `unpack-iterable` in it, since Python's operators (`+`,
             # etc.) can't unpack. An exception to this exception is that
             # tuple literals (`,`) can unpack. Finally, we allow unpacking in
             # `.` forms here so the user gets a better error message.
             sroot = ast_str(root)
-            if (sroot in _special_form_compilers or sroot in _bad_roots) and (
+
+            bad_root = sroot in _bad_roots or (sroot == ast_str("annotate*")
+                                               and not allow_annotation_expression)
+
+            if (sroot in _special_form_compilers or bad_root) and (
                     sroot in (mangle(","), mangle(".")) or
                     not any(is_unpack("iterable", x) for x in args)):
-                if sroot in _bad_roots:
+                if bad_root:
                     raise self._syntax_error(expr,
                         "The special form '{}' is not allowed here".format(root))
                 # `sroot` is a special operator. Get the build method and
@@ -1684,6 +1788,11 @@ class HyASTCompiler(object):
                                        attr=ast_str(root),
                                        ctx=ast.Load())
 
+        elif is_annotate_expression(root):
+            # Flatten and compile the annotation expression.
+            ann_expr = HyExpression(root + args).replace(root)
+            return self.compile_expression(ann_expr, allow_annotation_expression=True)
+
         if not func:
             func = self.compile(root)
 

hy/core/bootstrap.hy

@@ -60,14 +60,12 @@
 (defmacro macro-error [expression reason &optional [filename '--name--]]
   `(raise (hy.errors.HyMacroExpansionError ~reason ~filename ~expression None)))
 
-(defmacro defn [name lambda-list &rest body]
+(defmacro defn [name &rest args]
-  "Define `name` as a function with `lambda-list` signature and body `body`."
+  "Define `name` as a function with `args` as the signature, annotations, and body."
   (import hy)
   (if (not (= (type name) hy.HySymbol))
     (macro-error name "defn takes a name as first argument"))
-  (if (not (isinstance lambda-list hy.HyList))
+  `(setv ~name (fn* ~@args)))
-    (macro-error name "defn takes a parameter list as second argument"))
-  `(setv ~name (fn* ~lambda-list ~@body)))
 
 (defmacro defn/a [name lambda-list &rest body]
   "Define `name` as a function with `lambda-list` signature and body `body`."

hy/core/macros.hy

@@ -139,6 +139,20 @@ the second form, the second result is inserted into the third form, and so on."
   ret)
 
 
+(defmacro of [base &rest args]
+  "Shorthand for indexing for type annotations.
+
+If only one arguments are given, this expands to just that argument. If two arguments are
+given, it expands to indexing the first argument via the second. Otherwise, the first argument
+is indexed using a tuple of the rest.
+
+E.g. `(of List int)` -> `List[int]`, `(of Dict str str)` -> `Dict[str, str]`."
+  (if
+    (empty? args) base
+    (= (len args) 1) `(get ~base ~@args)
+    `(get ~base (, ~@args))))
+
+
 (defmacro if-not [test not-branch &optional yes-branch]
   "Like `if`, but execute the first branch when the test fails"
   `(if* (not ~test) ~not-branch ~yes-branch))

hy/lex/lexer.py

@@ -10,7 +10,8 @@ lg = LexerGenerator()
 
 # A regexp for something that should end a quoting/unquoting operator
 # i.e. a space or a closing brace/paren/curly
-end_quote = r'(?![\s\)\]\}])'
+end_quote_set = r'\s\)\]\}'
+end_quote = r'(?![%s])' % end_quote_set
 
 identifier = r'[^()\[\]{}\'"\s;]+'
 
@@ -25,6 +26,7 @@ lg.add('QUOTE', r'\'%s' % end_quote)
 lg.add('QUASIQUOTE', r'`%s' % end_quote)
 lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
 lg.add('UNQUOTE', r'~%s' % end_quote)
+lg.add('ANNOTATION', r'\^(?![=%s])' % end_quote_set)
 lg.add('DISCARD', r'#_')
 lg.add('HASHSTARS', r'#\*+')
 lg.add('BRACKETSTRING', r'''(?x)

hy/lex/parser.py

@@ -134,6 +134,12 @@ def term_unquote_splice(state, p):
     return HyExpression([HySymbol("unquote-splice"), p[1]])
 
 
+@pg.production("term : ANNOTATION term")
+@set_quote_boundaries
+def term_annotation(state, p):
+    return HyExpression([HySymbol("annotate*"), p[1]])
+
+
 @pg.production("term : HASHSTARS term")
 @set_quote_boundaries
 def term_hashstars(state, p):

tests/native_tests/language.hy

@@ -68,15 +68,15 @@
   "NATIVE: test that setv doesn't work on names Python can't assign to
   and that we can't mangle"
   (try (eval '(setv None 1))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(defn None [] (print "hello")))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(setv False 1))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(setv True 0))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(defn True [] (print "hello")))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e))))))
 
 
 (defn test-setv-pairs []
@@ -908,6 +908,22 @@
   (assert (= mooey.__name__ "mooey")))
 
 
+(defn test-defn-annotations []
+  "NATIVE: test that annotations in defn work"
+
+  (defn f [^int p1 p2 ^str p3 &optional ^str o1 ^int [o2 0]
+           &rest ^str rest &kwonly ^str k1 ^int [k2 0] &kwargs ^bool kwargs])
+
+  (assert (= (. f __annotations__ ["p1"]) int))
+  (assert (= (. f __annotations__ ["p3"]) str))
+  (assert (= (. f __annotations__ ["o1"]) str))
+  (assert (= (. f __annotations__ ["o2"]) int))
+  (assert (= (. f __annotations__ ["rest"]) str))
+  (assert (= (. f __annotations__ ["k1"]) str))
+  (assert (= (. f __annotations__ ["k2"]) int))
+  (assert (= (. f __annotations__ ["kwargs"]) bool)))
+
+
 (defn test-return []
 
   ; `return` in main line

tests/native_tests/py36_only_tests.hy

@@ -6,6 +6,7 @@
 ;; conftest.py skips this file when running on Python <3.6.
 
 (import [asyncio [get-event-loop sleep]])
+(import [typing [get-type-hints List Dict]])
 
 
 (defn run-coroutine [coro]
@@ -38,6 +39,20 @@
         (else (setv x (+ x 50))))
       (assert (= x 53)))))
 
+(defn test-variable-annotations []
+  (defclass AnnotationContainer []
+    (setv ^int x 1 y 2)
+    (^bool z))
+
+  (setv annotations (get-type-hints AnnotationContainer))
+  (assert (= (get annotations "x") int))
+  (assert (= (get annotations "z") bool)))
+
+(defn test-of []
+  (assert (= (of str) str))
+  (assert (= (of List int) (get List int)))
+  (assert (= (of Dict str str) (get Dict (, str str)))))
+
 (defn test-pep-487 []
   (defclass QuestBase []
     (defn --init-subclass-- [cls swallow &kwargs kwargs]