diff --git a/NEWS.rst b/NEWS.rst index 3c7db04..e5b1d69 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -17,6 +17,7 @@ New Features inline Python code. * All augmented assignment operators (except `%=` and `^=`) now allow more than two arguments. +* PEP 3107 and PEP 526 function and variable annotations are now supported. Other Breaking Changes ------------------------------ diff --git a/hy/compiler.py b/hy/compiler.py index d908282..2b8bb6e 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -23,6 +23,7 @@ import re import textwrap import pkgutil import traceback +import itertools import importlib import inspect import types @@ -337,6 +338,15 @@ def mklist(*items, **kwargs): return make_hy_model(HyList, items, kwargs.get('rest')) +# Parse an annotation setting. +OPTIONAL_ANNOTATION = maybe(pexpr(sym("annotate*") + FORM) >> (lambda x: x[0])) + + +def is_annotate_expression(model): + return (isinstance(model, HyExpression) and model and isinstance(model[0], HySymbol) + and model[0] == HySymbol("annotate*")) + + class HyASTCompiler(object): """A Hy-to-Python AST compiler""" @@ -1334,7 +1344,7 @@ class HyASTCompiler(object): return ret + asty.AugAssign( expr, target=target, value=ret.force_expr, op=op()) - @special("setv", [many(FORM + FORM)]) + @special("setv", [many(OPTIONAL_ANNOTATION + FORM + FORM)]) @special((PY38, "setx"), [times(1, 1, SYM + FORM)]) def compile_def_expression(self, expr, root, decls): if not decls: @@ -1343,21 +1353,43 @@ class HyASTCompiler(object): result = Result() is_assignment_expr = root == HySymbol("setx") for decl in decls: - result += self._compile_assign(*decl, is_assignment_expr=is_assignment_expr) + if is_assignment_expr: + ann = None + name, value = decl + else: + ann, name, value = decl + + result += self._compile_assign(ann, name, value, + is_assignment_expr=is_assignment_expr) return result - def _compile_assign(self, name, result, *, is_assignment_expr=False): + @special(["annotate*"], [FORM, FORM]) + def compile_basic_annotation(self, expr, root, ann, target): + return self._compile_assign(ann, target, None) - if name in [HySymbol(x) for x in ("None", "True", "False")]: - raise self._syntax_error(name, - "Can't assign to `{}'".format(name)) + def _compile_assign(self, ann, name, value, *, is_assignment_expr = False): + # Ensure that assignment expressions have a result and no annotation. + assert not is_assignment_expr or (value is not None and ann is None) - result = self.compile(result) ld_name = self.compile(name) - if isinstance(ld_name.expr, ast.Call): - raise self._syntax_error(name, - "Can't assign to a callable: `{}'".format(name)) + annotate_only = value is None + if annotate_only: + result = Result() + else: + result = self.compile(value) + + invalid_name = False + if ann is not None: + # An annotation / annotated assignment is more strict with the target expression. + invalid_name = not isinstance(ld_name.expr, (ast.Name, ast.Attribute, ast.Subscript)) + else: + invalid_name = (str(name) in ("None", "True", "False") + or isinstance(ld_name.expr, ast.Call)) + + if invalid_name: + raise self._syntax_error(name, "illegal target for {}".format( + "annotation" if annotate_only else "assignment")) if (result.temp_variables and isinstance(name, HySymbol) @@ -1368,12 +1400,27 @@ class HyASTCompiler(object): result.expr = None else: st_name = self._storeize(name, ld_name) - node = (asty.NamedExpr - if is_assignment_expr - else asty.Assign) + + if ann is not None: + ann_result = self.compile(ann) + result = ann_result + result + + if is_assignment_expr: + node = asty.NamedExpr + elif ann is not None: + if not PY36: + raise self._syntax_error(name, "Variable annotations are not supported on " + "Python <=3.6") + + node = lambda x, **kw: asty.AnnAssign(x, annotation=ann_result.force_expr, + simple=int(isinstance(name, HySymbol)), + **kw) + else: + node = asty.Assign + result += node( name if hasattr(name, "start_line") else result, - value=result.force_expr, + value=result.force_expr if not annotate_only else None, target=st_name, targets=[st_name]) return result @@ -1432,18 +1479,34 @@ class HyASTCompiler(object): # The starred version is for internal use (particularly, in the # definition of `defn`). It ensures that a FunctionDef is # produced rather than a Lambda. + OPTIONAL_ANNOTATION, brackets( - many(NASYM), - maybe(sym("&optional") + many(NASYM | brackets(SYM, FORM))), - maybe(sym("&rest") + NASYM), - maybe(sym("&kwonly") + many(NASYM | brackets(SYM, FORM))), - maybe(sym("&kwargs") + NASYM)), + many(OPTIONAL_ANNOTATION + NASYM), + maybe(sym("&optional") + many(OPTIONAL_ANNOTATION + + (NASYM | brackets(SYM, FORM)))), + maybe(sym("&rest") + OPTIONAL_ANNOTATION + NASYM), + maybe(sym("&kwonly") + many(OPTIONAL_ANNOTATION + + (NASYM | brackets(SYM, FORM)))), + maybe(sym("&kwargs") + OPTIONAL_ANNOTATION + NASYM)), many(FORM)]) - def compile_function_def(self, expr, root, params, body): + def compile_function_def(self, expr, root, returns, params, body): force_functiondef = root in ("fn*", "fn/a") node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef ret = Result() + # NOTE: Our evaluation order of return type annotations is + # different from Python: Python evalautes them after the argument + # annotations / defaults (as that's where they are in the source), + # but Hy evaluates them *first*, since here they come before the # + # argument list. Therefore, it would be more confusing for + # readability to evaluate them after like Python. + + ret = Result() + returns_ann = None + if returns is not None: + returns_result = self.compile(returns) + ret += returns_result + mandatory, optional, rest, kwonly, kwargs = params optional = optional or [] @@ -1469,7 +1532,7 @@ class HyASTCompiler(object): body = self._compile_branch(body) - if not force_functiondef and not body.stmts: + if not force_functiondef and not body.stmts and returns is None: return ret + asty.Lambda(expr, args=args, body=body.force_expr) if body.expr: @@ -1481,7 +1544,8 @@ class HyASTCompiler(object): name=name, args=args, body=body.stmts or [asty.Pass(expr)], - decorator_list=[]) + decorator_list=[], + returns=returns_result.force_expr if returns is not None else None) ast_name = asty.Name(expr, id=name, ctx=ast.Load()) ret += Result(expr=ast_name, temp_variables=[ast_name, ret.stmts[-1]]) @@ -1491,7 +1555,7 @@ class HyASTCompiler(object): args_ast = [] args_defaults = [] - for decl in decls: + for ann, decl in decls: default = None # funcparserlib will check to make sure that the only times we @@ -1503,6 +1567,12 @@ class HyASTCompiler(object): if implicit_default_none: default = HySymbol('None').replace(sym) + if ann is not None: + ret += self.compile(ann) + ann_ast = ret.force_expr + else: + ann_ast = None + if default is not None: ret += self.compile(default) args_defaults.append(ret.force_expr) @@ -1513,7 +1583,7 @@ class HyASTCompiler(object): # positional args. args_defaults.append(None) - args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=None)) + args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=ann_ast)) return args_ast, args_defaults, ret @@ -1564,12 +1634,22 @@ class HyASTCompiler(object): return expr new_args = [] - pairs = list(expr[1:]) - while pairs: - k, v = (pairs.pop(0), pairs.pop(0)) + decls = list(expr[1:]) + while decls: + if is_annotate_expression(decls[0]): + # Handle annotations. + ann = decls.pop(0) + else: + ann = None + + k, v = (decls.pop(0), decls.pop(0)) if ast_str(k) == "__init__" and isinstance(v, HyExpression): v += HyExpression([HySymbol("None")]) - new_args.extend([k, v]) + + if ann is not None: + new_args.append(ann) + + new_args.extend((k, v)) return HyExpression([HySymbol("setv")] + new_args).replace(expr) @special("dispatch-tag-macro", [STR, FORM]) @@ -1628,7 +1708,7 @@ class HyASTCompiler(object): return Result(stmts=o) if exec_mode else o @builds_model(HyExpression) - def compile_expression(self, expr): + def compile_expression(self, expr, *, allow_annotation_expression=False): # Perform macro expansions expr = macroexpand(expr, self.module, self) if not isinstance(expr, HyExpression): @@ -1644,17 +1724,20 @@ class HyASTCompiler(object): func = None if isinstance(root, HySymbol): - # First check if `root` is a special operator, unless it has an # `unpack-iterable` in it, since Python's operators (`+`, # etc.) can't unpack. An exception to this exception is that # tuple literals (`,`) can unpack. Finally, we allow unpacking in # `.` forms here so the user gets a better error message. sroot = ast_str(root) - if (sroot in _special_form_compilers or sroot in _bad_roots) and ( + + bad_root = sroot in _bad_roots or (sroot == ast_str("annotate*") + and not allow_annotation_expression) + + if (sroot in _special_form_compilers or bad_root) and ( sroot in (mangle(","), mangle(".")) or not any(is_unpack("iterable", x) for x in args)): - if sroot in _bad_roots: + if bad_root: raise self._syntax_error(expr, "The special form '{}' is not allowed here".format(root)) # `sroot` is a special operator. Get the build method and @@ -1705,6 +1788,11 @@ class HyASTCompiler(object): attr=ast_str(root), ctx=ast.Load()) + elif is_annotate_expression(root): + # Flatten and compile the annotation expression. + ann_expr = HyExpression(root + args).replace(root) + return self.compile_expression(ann_expr, allow_annotation_expression=True) + if not func: func = self.compile(root) diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy index d5b7e19..b8da6c7 100644 --- a/hy/core/bootstrap.hy +++ b/hy/core/bootstrap.hy @@ -60,14 +60,12 @@ (defmacro macro-error [expression reason &optional [filename '--name--]] `(raise (hy.errors.HyMacroExpansionError ~reason ~filename ~expression None))) -(defmacro defn [name lambda-list &rest body] - "Define `name` as a function with `lambda-list` signature and body `body`." +(defmacro defn [name &rest args] + "Define `name` as a function with `args` as the signature, annotations, and body." (import hy) (if (not (= (type name) hy.HySymbol)) (macro-error name "defn takes a name as first argument")) - (if (not (isinstance lambda-list hy.HyList)) - (macro-error name "defn takes a parameter list as second argument")) - `(setv ~name (fn* ~lambda-list ~@body))) + `(setv ~name (fn* ~@args))) (defmacro defn/a [name lambda-list &rest body] "Define `name` as a function with `lambda-list` signature and body `body`." diff --git a/hy/lex/lexer.py b/hy/lex/lexer.py index f202d94..fddfe4b 100755 --- a/hy/lex/lexer.py +++ b/hy/lex/lexer.py @@ -10,7 +10,8 @@ lg = LexerGenerator() # A regexp for something that should end a quoting/unquoting operator # i.e. a space or a closing brace/paren/curly -end_quote = r'(?![\s\)\]\}])' +end_quote_set = r'\s\)\]\}' +end_quote = r'(?![%s])' % end_quote_set identifier = r'[^()\[\]{}\'"\s;]+' @@ -25,6 +26,7 @@ lg.add('QUOTE', r'\'%s' % end_quote) lg.add('QUASIQUOTE', r'`%s' % end_quote) lg.add('UNQUOTESPLICE', r'~@%s' % end_quote) lg.add('UNQUOTE', r'~%s' % end_quote) +lg.add('ANNOTATION', r'\^(?![=%s])' % end_quote_set) lg.add('DISCARD', r'#_') lg.add('HASHSTARS', r'#\*+') lg.add('BRACKETSTRING', r'''(?x) diff --git a/hy/lex/parser.py b/hy/lex/parser.py index 1d5ce24..f3edb0e 100755 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -134,6 +134,12 @@ def term_unquote_splice(state, p): return HyExpression([HySymbol("unquote-splice"), p[1]]) +@pg.production("term : ANNOTATION term") +@set_quote_boundaries +def term_annotation(state, p): + return HyExpression([HySymbol("annotate*"), p[1]]) + + @pg.production("term : HASHSTARS term") @set_quote_boundaries def term_hashstars(state, p): diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 688f985..37dce9e 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -68,15 +68,15 @@ "NATIVE: test that setv doesn't work on names Python can't assign to and that we can't mangle" (try (eval '(setv None 1)) - (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e))))) (try (eval '(defn None [] (print "hello"))) - (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e))))) (try (eval '(setv False 1)) - (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e))))) (try (eval '(setv True 0)) - (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e))))) (try (eval '(defn True [] (print "hello"))) - (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))) + (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))) (defn test-setv-pairs [] @@ -908,6 +908,22 @@ (assert (= mooey.__name__ "mooey"))) +(defn test-defn-annotations [] + "NATIVE: test that annotations in defn work" + + (defn f [^int p1 p2 ^str p3 &optional ^str o1 ^int [o2 0] + &rest ^str rest &kwonly ^str k1 ^int [k2 0] &kwargs ^bool kwargs]) + + (assert (= (. f __annotations__ ["p1"]) int)) + (assert (= (. f __annotations__ ["p3"]) str)) + (assert (= (. f __annotations__ ["o1"]) str)) + (assert (= (. f __annotations__ ["o2"]) int)) + (assert (= (. f __annotations__ ["rest"]) str)) + (assert (= (. f __annotations__ ["k1"]) str)) + (assert (= (. f __annotations__ ["k2"]) int)) + (assert (= (. f __annotations__ ["kwargs"]) bool))) + + (defn test-return [] ; `return` in main line diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy index 3ba0c4f..99f2144 100644 --- a/tests/native_tests/py36_only_tests.hy +++ b/tests/native_tests/py36_only_tests.hy @@ -6,6 +6,7 @@ ;; conftest.py skips this file when running on Python <3.6. (import [asyncio [get-event-loop sleep]]) +(import [typing [get-type-hints]]) (defn run-coroutine [coro] @@ -38,6 +39,15 @@ (else (setv x (+ x 50)))) (assert (= x 53))))) +(defn test-variable-annotations [] + (defclass AnnotationContainer [] + (setv ^int x 1 y 2) + (^bool z)) + + (setv annotations (get-type-hints AnnotationContainer)) + (assert (= (get annotations "x") int)) + (assert (= (get annotations "z") bool))) + (defn test-pep-487 [] (defclass QuestBase [] (defn --init-subclass-- [cls swallow &kwargs kwargs]