Implement PEP 3107 & 526 annotations (closes #1794)

NEWS.rst

@@ -17,6 +17,7 @@ New Features
   inline Python code.
 * All augmented assignment operators (except `%=` and `^=`) now allow
   more than two arguments.
+* PEP 3107 and PEP 526 function and variable annotations are now supported.
 
 Other Breaking Changes
 ------------------------------

hy/compiler.py

@@ -23,6 +23,7 @@ import re
 import textwrap
 import pkgutil
 import traceback
+import itertools
 import importlib
 import inspect
 import types
@@ -337,6 +338,15 @@ def mklist(*items, **kwargs):
    return make_hy_model(HyList, items, kwargs.get('rest'))
 
 
+# Parse an annotation setting.
+OPTIONAL_ANNOTATION = maybe(pexpr(sym("annotate*") + FORM) >> (lambda x: x[0]))
+
+
+def is_annotate_expression(model):
+    return (isinstance(model, HyExpression) and model and isinstance(model[0], HySymbol)
+            and model[0] == HySymbol("annotate*"))
+
+
 class HyASTCompiler(object):
     """A Hy-to-Python AST compiler"""
 
@@ -1334,7 +1344,7 @@ class HyASTCompiler(object):
         return ret + asty.AugAssign(
             expr, target=target, value=ret.force_expr, op=op())
 
-    @special("setv", [many(FORM + FORM)])
+    @special("setv", [many(OPTIONAL_ANNOTATION + FORM + FORM)])
     @special((PY38, "setx"), [times(1, 1, SYM + FORM)])
     def compile_def_expression(self, expr, root, decls):
         if not decls:
@@ -1343,21 +1353,43 @@ class HyASTCompiler(object):
         result = Result()
         is_assignment_expr = root == HySymbol("setx")
         for decl in decls:
-            result += self._compile_assign(*decl, is_assignment_expr=is_assignment_expr)
+            if is_assignment_expr:
+                ann = None
+                name, value = decl
+            else:
+                ann, name, value = decl
+
+            result += self._compile_assign(ann, name, value,
+                                           is_assignment_expr=is_assignment_expr)
         return result
 
-    def _compile_assign(self, name, result, *, is_assignment_expr=False):
+    @special(["annotate*"], [FORM, FORM])
+    def compile_basic_annotation(self, expr, root, ann, target):
+        return self._compile_assign(ann, target, None)
 
-        if name in [HySymbol(x) for x in ("None", "True", "False")]:
+    def _compile_assign(self, ann, name, value, *, is_assignment_expr = False):
-            raise self._syntax_error(name,
+        # Ensure that assignment expressions have a result and no annotation.
-                "Can't assign to `{}'".format(name))
+        assert not is_assignment_expr or (value is not None and ann is None)
 
-        result = self.compile(result)
         ld_name = self.compile(name)
 
-        if isinstance(ld_name.expr, ast.Call):
+        annotate_only = value is None
-            raise self._syntax_error(name,
+        if annotate_only:
-                "Can't assign to a callable: `{}'".format(name))
+            result = Result()
+        else:
+            result = self.compile(value)
+
+        invalid_name = False
+        if ann is not None:
+            # An annotation / annotated assignment is more strict with the target expression.
+            invalid_name = not isinstance(ld_name.expr, (ast.Name, ast.Attribute, ast.Subscript))
+        else:
+            invalid_name = (str(name) in ("None", "True", "False")
+                            or isinstance(ld_name.expr, ast.Call))
+
+        if invalid_name:
+            raise self._syntax_error(name, "illegal target for {}".format(
+                                        "annotation" if annotate_only else "assignment"))
 
         if (result.temp_variables
                 and isinstance(name, HySymbol)
@@ -1368,12 +1400,27 @@ class HyASTCompiler(object):
                 result.expr = None
         else:
             st_name = self._storeize(name, ld_name)
-            node = (asty.NamedExpr
+
-                if is_assignment_expr
+            if ann is not None:
-                else asty.Assign)
+                ann_result = self.compile(ann)
+                result = ann_result + result
+
+            if is_assignment_expr:
+                node = asty.NamedExpr
+            elif ann is not None:
+                if not PY36:
+                    raise self._syntax_error(name, "Variable annotations are not supported on "
+                                                   "Python <=3.6")
+
+                node = lambda x, **kw: asty.AnnAssign(x, annotation=ann_result.force_expr,
+                                                      simple=int(isinstance(name, HySymbol)),
+                                                      **kw)
+            else:
+                node = asty.Assign
+
             result += node(
                 name if hasattr(name, "start_line") else result,
-                value=result.force_expr,
+                value=result.force_expr if not annotate_only else None,
                 target=st_name, targets=[st_name])
 
         return result
@@ -1432,18 +1479,34 @@ class HyASTCompiler(object):
         # The starred version is for internal use (particularly, in the
         # definition of `defn`). It ensures that a FunctionDef is
         # produced rather than a Lambda.
+        OPTIONAL_ANNOTATION,
         brackets(
-            many(NASYM),
+            many(OPTIONAL_ANNOTATION + NASYM),
-            maybe(sym("&optional") + many(NASYM | brackets(SYM, FORM))),
+            maybe(sym("&optional") + many(OPTIONAL_ANNOTATION
-            maybe(sym("&rest") + NASYM),
+                                            + (NASYM | brackets(SYM, FORM)))),
-            maybe(sym("&kwonly") + many(NASYM | brackets(SYM, FORM))),
+            maybe(sym("&rest") + OPTIONAL_ANNOTATION + NASYM),
-            maybe(sym("&kwargs") + NASYM)),
+            maybe(sym("&kwonly") + many(OPTIONAL_ANNOTATION
+                                        + (NASYM | brackets(SYM, FORM)))),
+            maybe(sym("&kwargs") + OPTIONAL_ANNOTATION + NASYM)),
         many(FORM)])
-    def compile_function_def(self, expr, root, params, body):
+    def compile_function_def(self, expr, root, returns, params, body):
         force_functiondef = root in ("fn*", "fn/a")
         node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef
         ret = Result()
 
+        # NOTE: Our evaluation order of return type annotations is
+        # different from Python: Python evalautes them after the argument
+        # annotations / defaults (as that's where they are in the source),
+        # but Hy evaluates them *first*, since here they come before the #
+        # argument list. Therefore, it would be more confusing for
+        # readability to evaluate them after like Python.
+
+        ret = Result()
+        returns_ann = None
+        if returns is not None:
+            returns_result = self.compile(returns)
+            ret += returns_result
+
         mandatory, optional, rest, kwonly, kwargs = params
 
         optional = optional or []
@@ -1469,7 +1532,7 @@ class HyASTCompiler(object):
 
         body = self._compile_branch(body)
 
-        if not force_functiondef and not body.stmts:
+        if not force_functiondef and not body.stmts and returns is None:
             return ret + asty.Lambda(expr, args=args, body=body.force_expr)
 
         if body.expr:
@@ -1481,7 +1544,8 @@ class HyASTCompiler(object):
                     name=name,
                     args=args,
                     body=body.stmts or [asty.Pass(expr)],
-                    decorator_list=[])
+                    decorator_list=[],
+                    returns=returns_result.force_expr if returns is not None else None)
 
         ast_name = asty.Name(expr, id=name, ctx=ast.Load())
         ret += Result(expr=ast_name, temp_variables=[ast_name, ret.stmts[-1]])
@@ -1491,7 +1555,7 @@ class HyASTCompiler(object):
         args_ast = []
         args_defaults = []
 
-        for decl in decls:
+        for ann, decl in decls:
             default = None
 
             # funcparserlib will check to make sure that the only times we
@@ -1503,6 +1567,12 @@ class HyASTCompiler(object):
                 if implicit_default_none:
                     default = HySymbol('None').replace(sym)
 
+            if ann is not None:
+                ret += self.compile(ann)
+                ann_ast = ret.force_expr
+            else:
+                ann_ast = None
+
             if default is not None:
                 ret += self.compile(default)
                 args_defaults.append(ret.force_expr)
@@ -1513,7 +1583,7 @@ class HyASTCompiler(object):
                 # positional args.
                 args_defaults.append(None)
 
-            args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=None))
+            args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=ann_ast))
 
         return args_ast, args_defaults, ret
 
@@ -1564,12 +1634,22 @@ class HyASTCompiler(object):
             return expr
 
         new_args = []
-        pairs = list(expr[1:])
+        decls = list(expr[1:])
-        while pairs:
+        while decls:
-            k, v = (pairs.pop(0), pairs.pop(0))
+            if is_annotate_expression(decls[0]):
+                # Handle annotations.
+                ann = decls.pop(0)
+            else:
+                ann = None
+
+            k, v = (decls.pop(0), decls.pop(0))
             if ast_str(k) == "__init__" and isinstance(v, HyExpression):
                 v += HyExpression([HySymbol("None")])
-            new_args.extend([k, v])
+
+            if ann is not None:
+                new_args.append(ann)
+
+            new_args.extend((k, v))
         return HyExpression([HySymbol("setv")] + new_args).replace(expr)
 
     @special("dispatch-tag-macro", [STR, FORM])
@@ -1628,7 +1708,7 @@ class HyASTCompiler(object):
         return Result(stmts=o) if exec_mode else o
 
     @builds_model(HyExpression)
-    def compile_expression(self, expr):
+    def compile_expression(self, expr, *, allow_annotation_expression=False):
         # Perform macro expansions
         expr = macroexpand(expr, self.module, self)
         if not isinstance(expr, HyExpression):
@@ -1644,17 +1724,20 @@ class HyASTCompiler(object):
         func = None
 
         if isinstance(root, HySymbol):
-
             # First check if `root` is a special operator, unless it has an
             # `unpack-iterable` in it, since Python's operators (`+`,
             # etc.) can't unpack. An exception to this exception is that
             # tuple literals (`,`) can unpack. Finally, we allow unpacking in
             # `.` forms here so the user gets a better error message.
             sroot = ast_str(root)
-            if (sroot in _special_form_compilers or sroot in _bad_roots) and (
+
+            bad_root = sroot in _bad_roots or (sroot == ast_str("annotate*")
+                                               and not allow_annotation_expression)
+
+            if (sroot in _special_form_compilers or bad_root) and (
                     sroot in (mangle(","), mangle(".")) or
                     not any(is_unpack("iterable", x) for x in args)):
-                if sroot in _bad_roots:
+                if bad_root:
                     raise self._syntax_error(expr,
                         "The special form '{}' is not allowed here".format(root))
                 # `sroot` is a special operator. Get the build method and
@@ -1705,6 +1788,11 @@ class HyASTCompiler(object):
                                        attr=ast_str(root),
                                        ctx=ast.Load())
 
+        elif is_annotate_expression(root):
+            # Flatten and compile the annotation expression.
+            ann_expr = HyExpression(root + args).replace(root)
+            return self.compile_expression(ann_expr, allow_annotation_expression=True)
+
         if not func:
             func = self.compile(root)
 

hy/core/bootstrap.hy

@@ -60,14 +60,12 @@
 (defmacro macro-error [expression reason &optional [filename '--name--]]
   `(raise (hy.errors.HyMacroExpansionError ~reason ~filename ~expression None)))
 
-(defmacro defn [name lambda-list &rest body]
+(defmacro defn [name &rest args]
-  "Define `name` as a function with `lambda-list` signature and body `body`."
+  "Define `name` as a function with `args` as the signature, annotations, and body."
   (import hy)
   (if (not (= (type name) hy.HySymbol))
     (macro-error name "defn takes a name as first argument"))
-  (if (not (isinstance lambda-list hy.HyList))
+  `(setv ~name (fn* ~@args)))
-    (macro-error name "defn takes a parameter list as second argument"))
-  `(setv ~name (fn* ~lambda-list ~@body)))
 
 (defmacro defn/a [name lambda-list &rest body]
   "Define `name` as a function with `lambda-list` signature and body `body`."

hy/lex/lexer.py

@@ -10,7 +10,8 @@ lg = LexerGenerator()
 
 # A regexp for something that should end a quoting/unquoting operator
 # i.e. a space or a closing brace/paren/curly
-end_quote = r'(?![\s\)\]\}])'
+end_quote_set = r'\s\)\]\}'
+end_quote = r'(?![%s])' % end_quote_set
 
 identifier = r'[^()\[\]{}\'"\s;]+'
 
@@ -25,6 +26,7 @@ lg.add('QUOTE', r'\'%s' % end_quote)
 lg.add('QUASIQUOTE', r'`%s' % end_quote)
 lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
 lg.add('UNQUOTE', r'~%s' % end_quote)
+lg.add('ANNOTATION', r'\^(?![=%s])' % end_quote_set)
 lg.add('DISCARD', r'#_')
 lg.add('HASHSTARS', r'#\*+')
 lg.add('BRACKETSTRING', r'''(?x)

hy/lex/parser.py

@@ -134,6 +134,12 @@ def term_unquote_splice(state, p):
     return HyExpression([HySymbol("unquote-splice"), p[1]])
 
 
+@pg.production("term : ANNOTATION term")
+@set_quote_boundaries
+def term_annotation(state, p):
+    return HyExpression([HySymbol("annotate*"), p[1]])
+
+
 @pg.production("term : HASHSTARS term")
 @set_quote_boundaries
 def term_hashstars(state, p):

tests/native_tests/language.hy

@@ -68,15 +68,15 @@
   "NATIVE: test that setv doesn't work on names Python can't assign to
   and that we can't mangle"
   (try (eval '(setv None 1))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(defn None [] (print "hello")))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(setv False 1))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(setv True 0))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(defn True [] (print "hello")))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e))))))
 
 
 (defn test-setv-pairs []
@@ -908,6 +908,22 @@
   (assert (= mooey.__name__ "mooey")))
 
 
+(defn test-defn-annotations []
+  "NATIVE: test that annotations in defn work"
+
+  (defn f [^int p1 p2 ^str p3 &optional ^str o1 ^int [o2 0]
+           &rest ^str rest &kwonly ^str k1 ^int [k2 0] &kwargs ^bool kwargs])
+
+  (assert (= (. f __annotations__ ["p1"]) int))
+  (assert (= (. f __annotations__ ["p3"]) str))
+  (assert (= (. f __annotations__ ["o1"]) str))
+  (assert (= (. f __annotations__ ["o2"]) int))
+  (assert (= (. f __annotations__ ["rest"]) str))
+  (assert (= (. f __annotations__ ["k1"]) str))
+  (assert (= (. f __annotations__ ["k2"]) int))
+  (assert (= (. f __annotations__ ["kwargs"]) bool)))
+
+
 (defn test-return []
 
   ; `return` in main line

tests/native_tests/py36_only_tests.hy

@@ -6,6 +6,7 @@
 ;; conftest.py skips this file when running on Python <3.6.
 
 (import [asyncio [get-event-loop sleep]])
+(import [typing [get-type-hints]])
 
 
 (defn run-coroutine [coro]
@@ -38,6 +39,15 @@
         (else (setv x (+ x 50))))
       (assert (= x 53)))))
 
+(defn test-variable-annotations []
+  (defclass AnnotationContainer []
+    (setv ^int x 1 y 2)
+    (^bool z))
+
+  (setv annotations (get-type-hints AnnotationContainer))
+  (assert (= (get annotations "x") int))
+  (assert (= (get annotations "z") bool)))
+
 (defn test-pep-487 []
   (defclass QuestBase []
     (defn --init-subclass-- [cls swallow &kwargs kwargs]