From 902926c543fa57a91b756e95168ded9543660c8c Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 8 Dec 2018 15:36:13 -0500 Subject: [PATCH 1/8] Use ._syntax_error in place of HyTypeError. And standardize the indentation of these calls. --- hy/compiler.py | 96 ++++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 50 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index f3b55a3..586129e 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -436,6 +436,9 @@ class HyASTCompiler(object): except Exception as e: raise_empty(HyCompileError, e, sys.exc_info()[2]) + def _syntax_error(self, expr, message): + return HyTypeError(expr, message) + def _compile_collect(self, exprs, with_kwargs=False, dict_display=False, oldpy_unpack=False): """Collect the expression contexts from a list of compiled expression. @@ -455,8 +458,8 @@ class HyASTCompiler(object): if not PY35 and oldpy_unpack and is_unpack("iterable", expr): if oldpy_starargs: - raise HyTypeError(expr, "Pythons < 3.5 allow only one " - "`unpack-iterable` per call") + raise self._syntax_error(expr, + "Pythons < 3.5 allow only one `unpack-iterable` per call") oldpy_starargs = self.compile(expr[1]) ret += oldpy_starargs oldpy_starargs = oldpy_starargs.force_expr @@ -472,21 +475,20 @@ class HyASTCompiler(object): expr, arg=None, value=ret.force_expr)) elif oldpy_unpack: if oldpy_kwargs: - raise HyTypeError(expr, "Pythons < 3.5 allow only one " - "`unpack-mapping` per call") + raise self._syntax_error(expr, + "Pythons < 3.5 allow only one `unpack-mapping` per call") oldpy_kwargs = ret.force_expr elif with_kwargs and isinstance(expr, HyKeyword): try: value = next(exprs_iter) except StopIteration: - raise HyTypeError(expr, - "Keyword argument {kw} needs " - "a value.".format(kw=expr)) + raise self._syntax_error(expr, + "Keyword argument {kw} needs a value.".format(kw=expr)) if not expr: - raise HyTypeError(expr, "Can't call a function with the " - "empty keyword") + raise self._syntax_error(expr, + "Can't call a function with the empty keyword") compiled_value = self.compile(value) ret += compiled_value @@ -527,8 +529,8 @@ class HyASTCompiler(object): if isinstance(name, Result): if not name.is_expr(): - raise HyTypeError(expr, - "Can't assign or delete a non-expression") + raise self._syntax_error(expr, + "Can't assign or delete a non-expression") name = name.expr if isinstance(name, (ast.Tuple, ast.List)): @@ -547,9 +549,8 @@ class HyASTCompiler(object): new_name = ast.Starred( value=self._storeize(expr, name.value, func)) else: - raise HyTypeError(expr, - "Can't assign or delete a %s" % - type(expr).__name__) + raise self._syntax_error(expr, + "Can't assign or delete a %s" % type(expr).__name__) new_name.ctx = func() ast.copy_location(new_name, name) @@ -575,9 +576,8 @@ class HyASTCompiler(object): op = unmangle(ast_str(form[0])) if level == 0 and op in ("unquote", "unquote-splice"): if len(form) != 2: - raise HyTypeError(form, - ("`%s' needs 1 argument, got %s" % - op, len(form) - 1)) + raise HyTypeError("`%s' needs 1 argument, got %s" % op, len(form) - 1, + self.filename, form, self.source) return set(), form[1], op == "unquote-splice" elif op == "quasiquote": level += 1 @@ -629,7 +629,8 @@ class HyASTCompiler(object): @special("unpack-iterable", [FORM]) def compile_unpack_iterable(self, expr, root, arg): if not PY3: - raise HyTypeError(expr, "`unpack-iterable` isn't allowed here") + raise self._syntax_error(expr, + "`unpack-iterable` isn't allowed here") ret = self.compile(arg) ret += asty.Starred(expr, value=ret.force_expr, ctx=ast.Load()) return ret @@ -659,7 +660,8 @@ class HyASTCompiler(object): if cause is not None: if not PY3: - raise HyTypeError(expr, "raise from only supported in python 3") + raise self._syntax_error(expr, + "raise from only supported in python 3") cause = self.compile(cause) ret += cause cause = cause.force_expr @@ -706,13 +708,11 @@ class HyASTCompiler(object): # Using (else) without (except) is verboten! if orelse and not handlers: - raise HyTypeError( - expr, + raise self._syntax_error(expr, "`try' cannot have `else' without `except'") # Likewise a bare (try) or (try BODY). if not (handlers or finalbody): - raise HyTypeError( - expr, + raise self._syntax_error(expr, "`try' must have an `except' or `finally' clause") returnable = Result( @@ -963,7 +963,8 @@ class HyASTCompiler(object): def compile_decorate_expression(self, expr, name, args): decs, fn = args[:-1], self.compile(args[-1]) if not fn.stmts or not isinstance(fn.stmts[-1], _decoratables): - raise HyTypeError(args[-1], "Decorated a non-function") + raise self._syntax_error(args[-1], + "Decorated a non-function") decs, ret, _ = self._compile_collect(decs) fn.stmts[-1].decorator_list = decs + fn.stmts[-1].decorator_list return ret + fn @@ -1194,8 +1195,8 @@ class HyASTCompiler(object): if (HySymbol('*'), None) in kids: if len(kids) != 1: star = kids[kids.index((HySymbol('*'), None))][0] - raise HyTypeError(star, "* in an import name list " - "must be on its own") + raise self._syntax_error(star, + "* in an import name list must be on its own") else: assignments = [(k, v or k) for k, v in kids] @@ -1390,15 +1391,15 @@ class HyASTCompiler(object): if str_name in (["None"] + (["True", "False"] if PY3 else [])): # Python 2 allows assigning to True and False, although # this is rarely wise. - raise HyTypeError(name, - "Can't assign to `%s'" % str_name) + raise self._syntax_error(name, + "Can't assign to `%s'" % str_name) result = self.compile(result) ld_name = self.compile(name) if isinstance(ld_name.expr, ast.Call): - raise HyTypeError(name, - "Can't assign to a callable: `%s'" % str_name) + raise self._syntax_error(name, + "Can't assign to a callable: `%s'" % str_name) if (result.temp_variables and isinstance(name, HySymbol) @@ -1474,7 +1475,8 @@ class HyASTCompiler(object): mandatory, optional, rest, kwonly, kwargs = params optional, defaults, ret = self._parse_optional_args(optional) if kwonly is not None and not PY3: - raise HyTypeError(params, "&kwonly parameters require Python 3") + raise self._syntax_error(params, + "&kwonly parameters require Python 3") kwonly, kw_defaults, ret2 = self._parse_optional_args(kwonly, True) ret += ret2 main_args = mandatory + optional @@ -1627,8 +1629,8 @@ class HyASTCompiler(object): return self.compile(expr) if not expr: - raise HyTypeError( - expr, "empty expressions are not allowed at top level") + raise self._syntax_error(expr, + "empty expressions are not allowed at top level") args = list(expr) root = args.pop(0) @@ -1646,8 +1648,7 @@ class HyASTCompiler(object): sroot in (mangle(","), mangle(".")) or not any(is_unpack("iterable", x) for x in args)): if sroot in _bad_roots: - raise HyTypeError( - expr, + raise self._syntax_error(expr, "The special form '{}' is not allowed here".format(root)) # `sroot` is a special operator. Get the build method and # pattern-match the arguments. @@ -1655,11 +1656,10 @@ class HyASTCompiler(object): try: parse_tree = pattern.parse(args) except NoParseError as e: - raise HyTypeError( + raise self._syntax_error( expr[min(e.state.pos + 1, len(expr) - 1)], "parse error for special form '{}': {}".format( - root, - e.msg.replace("", "end of form"))) + root, e.msg.replace("", "end of form"))) return Result() + build_method( self, expr, unmangle(sroot), *parse_tree) @@ -1681,13 +1681,13 @@ class HyASTCompiler(object): FORM + many(FORM)).parse(args) except NoParseError: - raise HyTypeError( - expr, "attribute access requires object") + raise self._syntax_error(expr, + "attribute access requires object") # Reconstruct `args` to exclude `obj`. args = [x for p in kws for x in p] + list(rest) if is_unpack("iterable", obj): - raise HyTypeError( - obj, "can't call a method on an unpacking form") + raise self._syntax_error(obj, + "can't call a method on an unpacking form") func = self.compile(HyExpression( [HySymbol(".").replace(root), obj] + attrs)) @@ -1725,16 +1725,12 @@ class HyASTCompiler(object): glob, local = symbol.rsplit(".", 1) if not glob: - raise HyTypeError(symbol, 'cannot access attribute on ' - 'anything other than a name ' - '(in order to get attributes of ' - 'expressions, use ' - '`(. {attr})` or ' - '`(.{attr} )`)'.format( - attr=local)) + raise self._syntax_error(symbol, + 'cannot access attribute on anything other than a name (in order to get attributes of expressions, use `(. {attr})` or `(.{attr} )`)'.format(attr=local)) if not local: - raise HyTypeError(symbol, 'cannot access empty attribute') + raise self._syntax_error(symbol, + 'cannot access empty attribute') glob = HySymbol(glob).replace(symbol) ret = self.compile_symbol(glob) From 51c7efe6e8f3b69c9d0eb1a316e11ec228cfd159 Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Sun, 28 Oct 2018 20:43:17 -0500 Subject: [PATCH 2/8] Retain compiled source and file information for exceptions This commit refactors the exception/error classes and their handling. It also retains Hy source strings and their originating file information, when available, all throughout the core parser and compiler functions. As well, with these changes, calling code is no longer responsible for providing source and file details to exceptions, Closes hylang/hy#657. --- hy/_compat.py | 36 ++++++- hy/cmdline.py | 46 +++++---- hy/compiler.py | 170 ++++++++++++++++++++++++------- hy/core/bootstrap.hy | 20 ++-- hy/errors.py | 174 +++++++++++++++++++++++++------- hy/importer.py | 39 +++---- hy/lex/__init__.py | 51 +++++++--- hy/lex/exceptions.py | 59 ++++------- hy/lex/parser.py | 103 +++++++++---------- hy/macros.py | 50 +++++---- tests/compilers/test_ast.py | 6 +- tests/importer/test_importer.py | 25 ++--- tests/test_bin.py | 2 +- tests/test_lex.py | 22 +++- 14 files changed, 525 insertions(+), 278 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index bd9390f..55180ca 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -6,7 +6,7 @@ try: import __builtin__ as builtins except ImportError: import builtins # NOQA -import sys, keyword +import sys, keyword, textwrap PY3 = sys.version_info[0] >= 3 PY35 = sys.version_info >= (3, 5) @@ -22,11 +22,39 @@ bytes_type = bytes if PY3 else str # NOQA long_type = int if PY3 else long # NOQA string_types = str if PY3 else basestring # NOQA +# +# Inspired by the same-named `six` functions. +# if PY3: - exec('def raise_empty(t, *args): raise t(*args) from None') + raise_src = textwrap.dedent(''' + def raise_from(value, from_value): + try: + raise value from from_value + finally: + traceback = None + ''') + + def reraise(exc_type, value, traceback=None): + try: + raise value.with_traceback(traceback) + finally: + traceback = None + else: - def raise_empty(t, *args): - raise t(*args) + def raise_from(value, from_value=None): + raise value + + raise_src = textwrap.dedent(''' + def reraise(exc_type, value, traceback=None): + try: + raise exc_type, value, traceback + finally: + traceback = None + ''') + +raise_code = compile(raise_src, __file__, 'exec') +exec(raise_code) + def isidentifier(x): if x in ('True', 'False', 'None', 'print'): diff --git a/hy/cmdline.py b/hy/cmdline.py index 4c2e1c3..70d7b72 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -19,9 +19,9 @@ import astor.code_gen import hy from hy.lex import hy_parse, mangle -from hy.lex.exceptions import LexException, PrematureEndOfInput +from hy.lex.exceptions import PrematureEndOfInput from hy.compiler import HyASTCompiler, hy_compile, hy_eval -from hy.errors import HyTypeError +from hy.errors import HyTypeError, HyLanguageError, HySyntaxError from hy.importer import runhy from hy.completer import completion, Completer from hy.macros import macro, require @@ -101,15 +101,11 @@ class HyREPL(code.InteractiveConsole, object): self.showtraceback() try: - try: - do = hy_parse(source) - except PrematureEndOfInput: - return True - except LexException as e: - if e.source is None: - e.source = source - e.filename = filename - error_handler(e, use_simple_traceback=True) + do = hy_parse(source, filename=filename) + except PrematureEndOfInput: + return True + except HySyntaxError as e: + error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS) return False try: @@ -121,9 +117,12 @@ class HyREPL(code.InteractiveConsole, object): [ast.Expr(expr_ast.body)]) print(astor.to_source(new_ast)) - value = hy_eval(do, self.locals, + value = hy_eval(do, self.locals, self.module, ast_callback=ast_callback, - compiler=self.hy_compiler) + compiler=self.hy_compiler, + filename=filename, + source=source) + except HyTypeError as e: if e.source is None: e.source = source @@ -131,7 +130,7 @@ class HyREPL(code.InteractiveConsole, object): error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS) return False except Exception as e: - error_handler(e) + error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS) return False if value is not None: @@ -208,17 +207,19 @@ SIMPLE_TRACEBACKS = True def pretty_error(func, *args, **kw): try: return func(*args, **kw) - except (HyTypeError, LexException) as e: + except HyLanguageError as e: if SIMPLE_TRACEBACKS: print(e, file=sys.stderr) sys.exit(1) raise -def run_command(source): - tree = hy_parse(source) - require("hy.cmdline", "__main__", assignments="ALL") - pretty_error(hy_eval, tree, None, importlib.import_module('__main__')) +def run_command(source, filename=None): + tree = hy_parse(source, filename=filename) + __main__ = importlib.import_module('__main__') + require("hy.cmdline", __main__, assignments="ALL") + pretty_error(hy_eval, tree, None, __main__, filename=filename, + source=source) return 0 @@ -340,7 +341,7 @@ def cmdline_handler(scriptname, argv): if options.command: # User did "hy -c ..." - return run_command(options.command) + return run_command(options.command, filename='') if options.mod: # User did "hy -m ..." @@ -356,7 +357,7 @@ def cmdline_handler(scriptname, argv): if options.args: if options.args[0] == "-": # Read the program from stdin - return run_command(sys.stdin.read()) + return run_command(sys.stdin.read(), filename='') else: # User did "hy " @@ -447,11 +448,12 @@ def hy2py_main(): if options.FILE is None or options.FILE == '-': source = sys.stdin.read() + hst = pretty_error(hy_parse, source, filename='') else: with io.open(options.FILE, 'r', encoding='utf-8') as source_file: source = source_file.read() + hst = hy_parse(source, filename=options.FILE) - hst = pretty_error(hy_parse, source) if options.with_source: # need special printing on Windows in case the # codepage doesn't support utf-8 characters diff --git a/hy/compiler.py b/hy/compiler.py index 586129e..5377c33 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -9,20 +9,21 @@ from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex, from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole, notpexpr, dolike, pexpr, times, Tag, tag, unpack) from funcparserlib.parser import some, many, oneplus, maybe, NoParseError -from hy.errors import HyCompileError, HyTypeError +from hy.errors import (HyCompileError, HyTypeError, HyEvalError, + HyInternalError) from hy.lex import mangle, unmangle -from hy._compat import (str_type, string_types, bytes_type, long_type, PY3, - PY35, raise_empty) +from hy._compat import (string_types, str_type, bytes_type, long_type, PY3, + PY35, reraise) from hy.macros import require, load_macros, macroexpand, tag_macroexpand import hy.core +import pkgutil import traceback import importlib import inspect -import pkgutil import types import ast import sys @@ -340,22 +341,33 @@ def is_unpack(kind, x): class HyASTCompiler(object): """A Hy-to-Python AST compiler""" - def __init__(self, module): + def __init__(self, module, filename=None, source=None): """ Parameters ---------- module: str or types.ModuleType - Module in which the Hy tree is evaluated. + Module name or object in which the Hy tree is evaluated. + filename: str, optional + The name of the file for the source to be compiled. + This is optional information for informative error messages and + debugging. + source: str, optional + The source for the file, if any, being compiled. This is optional + information for informative error messages and debugging. """ self.anon_var_count = 0 self.imports = defaultdict(set) self.temp_if = None if not inspect.ismodule(module): - module = importlib.import_module(module) + self.module = importlib.import_module(module) + else: + self.module = module - self.module = module - self.module_name = module.__name__ + self.module_name = self.module.__name__ + + self.filename = filename + self.source = source # Hy expects these to be present, so we prep the module for Hy # compilation. @@ -431,13 +443,15 @@ class HyASTCompiler(object): # nested; so let's re-raise this exception, let's not wrap it in # another HyCompileError! raise - except HyTypeError: - raise + except HyTypeError as e: + reraise(type(e), e, None) except Exception as e: - raise_empty(HyCompileError, e, sys.exc_info()[2]) + f_exc = traceback.format_exc() + exc_msg = "Internal Compiler Bug 😱\n⤷ {}".format(f_exc) + reraise(HyCompileError, HyCompileError(exc_msg), sys.exc_info()[2]) def _syntax_error(self, expr, message): - return HyTypeError(expr, message) + return HyTypeError(message, self.filename, expr, self.source) def _compile_collect(self, exprs, with_kwargs=False, dict_display=False, oldpy_unpack=False): @@ -1614,7 +1628,29 @@ class HyASTCompiler(object): def compile_eval_and_compile(self, expr, root, body): new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr) - hy_eval(new_expr + body, self.module.__dict__, self.module) + try: + hy_eval(new_expr + body, + self.module.__dict__, + self.module, + filename=self.filename, + source=self.source) + except HyInternalError: + # Unexpected "meta" compilation errors need to be treated + # like normal (unexpected) compilation errors at this level + # (or the compilation level preceding this one). + raise + except Exception as e: + # These could be expected Hy language errors (e.g. syntax errors) + # or regular Python runtime errors that do not signify errors in + # the compilation *process* (although compilation did technically + # fail). + # We wrap these exceptions and pass them through. + reraise(HyEvalError, + HyEvalError(str(e), + self.filename, + body, + self.source), + sys.exc_info()[2]) return (self._compile_branch(body) if ast_str(root) == "eval_and_compile" @@ -1798,8 +1834,13 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False): def hy_eval(hytree, locals=None, module=None, ast_callback=None, - compiler=None): + compiler=None, filename='', source=None): """Evaluates a quoted expression and returns the value. + + If you're evaluating hand-crafted AST trees, make sure the line numbers + are set properly. Try `fix_missing_locations` and related functions in the + Python `ast` library. + Examples -------- => (eval '(print "Hello World")) @@ -1812,8 +1853,8 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None, Parameters ---------- - hytree: a Hy expression tree - Source code to parse. + hytree: HyObject + The Hy AST object to evaluate. locals: dict, optional Local environment in which to evaluate the Hy tree. Defaults to the @@ -1835,6 +1876,19 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None, An existing Hy compiler to use for compilation. Also serves as the `module` value when given. + filename: str, optional + The filename corresponding to the source for `tree`. This will be + overridden by the `filename` field of `tree`, if any; otherwise, it + defaults to "". When `compiler` is given, its `filename` field + value is always used. + + source: str, optional + A string containing the source code for `tree`. This will be + overridden by the `source` field of `tree`, if any; otherwise, + if `None`, an attempt will be made to obtain it from the module given by + `module`. When `compiler` is given, its `source` field value is always + used. + Returns ------- out : Result of evaluating the Hy compiled tree. @@ -1849,36 +1903,53 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None, if not isinstance(locals, dict): raise TypeError("Locals must be a dictionary") - _ast, expr = hy_compile(hytree, module=module, get_expr=True, - compiler=compiler) + # Does the Hy AST object come with its own information? + filename = getattr(hytree, 'filename', filename) or '' + source = getattr(hytree, 'source', source) - # Spoof the positions in the generated ast... - for node in ast.walk(_ast): - node.lineno = 1 - node.col_offset = 1 - - for node in ast.walk(expr): - node.lineno = 1 - node.col_offset = 1 + _ast, expr = hy_compile(hytree, module, get_expr=True, + compiler=compiler, filename=filename, + source=source) if ast_callback: ast_callback(_ast, expr) - globals = module.__dict__ - # Two-step eval: eval() the body of the exec call - eval(ast_compile(_ast, "", "exec"), globals, locals) + eval(ast_compile(_ast, filename, "exec"), + module.__dict__, locals) # Then eval the expression context and return that - return eval(ast_compile(expr, "", "eval"), globals, locals) + return eval(ast_compile(expr, filename, "eval"), + module.__dict__, locals) -def hy_compile(tree, module=None, root=ast.Module, get_expr=False, - compiler=None): - """Compile a Hy tree into a Python AST tree. +def _module_file_source(module_name, filename, source): + """Try to obtain missing filename and source information from a module name + without actually loading the module. + """ + if filename is None or source is None: + mod_loader = pkgutil.get_loader(module_name) + if mod_loader: + if filename is None: + filename = mod_loader.get_filename(module_name) + if source is None: + source = mod_loader.get_source(module_name) + + # We need a non-None filename. + filename = filename or '' + + return filename, source + + +def hy_compile(tree, module, root=ast.Module, get_expr=False, + compiler=None, filename=None, source=None): + """Compile a HyObject tree into a Python AST Module. Parameters ---------- + tree: HyObject + The Hy AST object to compile. + module: str or types.ModuleType, optional Module, or name of the module, in which the Hy tree is evaluated. The module associated with `compiler` takes priority over this value. @@ -1893,18 +1964,43 @@ def hy_compile(tree, module=None, root=ast.Module, get_expr=False, An existing Hy compiler to use for compilation. Also serves as the `module` value when given. + filename: str, optional + The filename corresponding to the source for `tree`. This will be + overridden by the `filename` field of `tree`, if any; otherwise, it + defaults to "". When `compiler` is given, its `filename` field + value is always used. + + source: str, optional + A string containing the source code for `tree`. This will be + overridden by the `source` field of `tree`, if any; otherwise, + if `None`, an attempt will be made to obtain it from the module given by + `module`. When `compiler` is given, its `source` field value is always + used. + Returns ------- out : A Python AST tree """ module = get_compiler_module(module, compiler, False) + if isinstance(module, string_types): + if module.startswith('<') and module.endswith('>'): + module = types.ModuleType(module) + else: + module = importlib.import_module(ast_str(module, piecewise=True)) + + if not inspect.ismodule(module): + raise TypeError('Invalid module type: {}'.format(type(module))) + + filename = getattr(tree, 'filename', filename) + source = getattr(tree, 'source', source) + tree = wrap_value(tree) if not isinstance(tree, HyObject): - raise HyCompileError("`tree` must be a HyObject or capable of " - "being promoted to one") + raise TypeError("`tree` must be a HyObject or capable of " + "being promoted to one") - compiler = compiler or HyASTCompiler(module) + compiler = compiler or HyASTCompiler(module, filename=filename, source=source) result = compiler.compile(tree) expr = result.force_expr diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy index fa613a6..0c98f60 100644 --- a/hy/core/bootstrap.hy +++ b/hy/core/bootstrap.hy @@ -14,15 +14,14 @@ (if* (not (isinstance macro-name hy.models.HySymbol)) (raise (hy.errors.HyTypeError - macro-name (% "received a `%s' instead of a symbol for macro name" - (. (type name) - __name__))))) + (. (type name) --name--)) + --file-- macro-name None))) (for [kw '[&kwonly &kwargs]] (if* (in kw lambda-list) - (raise (hy.errors.HyTypeError macro-name - (% "macros cannot use %s" - kw))))) + (raise (hy.errors.HyTypeError (% "macros cannot use %s" + kw) + --file-- macro-name None)))) ;; this looks familiar... `(eval-and-compile (import hy) @@ -45,9 +44,9 @@ (if (and (not (isinstance tag-name hy.models.HySymbol)) (not (isinstance tag-name hy.models.HyString))) (raise (hy.errors.HyTypeError - tag-name (% "received a `%s' instead of a symbol for tag macro name" - (. (type tag-name) __name__))))) + (. (type tag-name) --name--)) + --file-- tag-name None))) (if (or (= tag-name ":") (= tag-name "&")) (raise (NameError (% "%s can't be used as a tag macro name" tag-name)))) @@ -58,9 +57,8 @@ ((hy.macros.tag ~tag-name) (fn ~lambda-list ~@body)))) -(defmacro macro-error [location reason] - "Error out properly within a macro at `location` giving `reason`." - `(raise (hy.errors.HyMacroExpansionError ~location ~reason))) +(defmacro macro-error [expression reason &optional [filename '--name--]] + `(raise (hy.errors.HyMacroExpansionError ~reason ~filename ~expression None))) (defmacro defn [name lambda-list &rest body] "Define `name` as a function with `lambda-list` signature and body `body`." diff --git a/hy/errors.py b/hy/errors.py index 7d36ab2..e8e9f98 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -5,41 +5,65 @@ import traceback +from functools import reduce + from clint.textui import colored class HyError(Exception): - """ - Generic Hy error. All internal Exceptions will be subclassed from this - Exception. - """ - pass - - -class HyCompileError(HyError): - def __init__(self, exception, traceback=None): - self.exception = exception - self.traceback = traceback - - def __str__(self): - if isinstance(self.exception, HyTypeError): - return str(self.exception) - if self.traceback: - tb = "".join(traceback.format_tb(self.traceback)).strip() - else: - tb = "No traceback available. 😟" - return("Internal Compiler Bug 😱\n⤷ %s: %s\nCompilation traceback:\n%s" - % (self.exception.__class__.__name__, - self.exception, tb)) - - -class HyTypeError(TypeError): - def __init__(self, expression, message): - super(HyTypeError, self).__init__(message) - self.expression = expression + def __init__(self, message, *args): self.message = message - self.source = None - self.filename = None + super(HyError, self).__init__(message, *args) + + +class HyInternalError(HyError): + """Unexpected errors occurring during compilation or parsing of Hy code. + + Errors sub-classing this are not intended to be user-facing, and will, + hopefully, never be seen by users! + """ + + def __init__(self, message, *args): + super(HyInternalError, self).__init__(message, *args) + + +class HyLanguageError(HyError): + """Errors caused by invalid use of the Hy language. + + This, and any errors inheriting from this, are user-facing. + """ + + def __init__(self, message, *args): + super(HyLanguageError, self).__init__(message, *args) + + +class HyCompileError(HyInternalError): + """Unexpected errors occurring within the compiler.""" + + +class HyTypeError(HyLanguageError, TypeError): + """TypeErrors occurring during the normal use of Hy.""" + + def __init__(self, message, filename=None, expression=None, source=None): + """ + Parameters + ---------- + message: str + The message to display for this error. + filename: str, optional + The filename for the source code generating this error. + expression: HyObject, optional + The Hy expression generating this error. + source: str, optional + The actual source code generating this error. + """ + self.message = message + self.filename = filename + self.expression = expression + self.source = source + + super(HyTypeError, self).__init__(message, filename, expression, + source) def __str__(self): @@ -93,12 +117,92 @@ class HyTypeError(TypeError): class HyMacroExpansionError(HyTypeError): - pass + """Errors caused by invalid use of Hy macros. + + This, and any errors inheriting from this, are user-facing. + """ -class HyIOError(HyError, IOError): +class HyEvalError(HyLanguageError): + """Errors occurring during code evaluation at compile-time. + + These errors distinguish unexpected errors within the compilation process + (i.e. `HyInternalError`s) from unrelated errors in user code evaluated by + the compiler (e.g. in `eval-and-compile`). + + This, and any errors inheriting from this, are user-facing. """ - Trivial subclass of IOError and HyError, to distinguish between - IOErrors raised by Hy itself as opposed to Hy programs. + + +class HyIOError(HyInternalError, IOError): + """ Subclass used to distinguish between IOErrors raised by Hy itself as + opposed to Hy programs. """ - pass + + +class HySyntaxError(HyLanguageError, SyntaxError): + """Error during the Lexing of a Hython expression.""" + + def __init__(self, message, filename=None, lineno=-1, colno=-1, + source=None): + """ + Parameters + ---------- + message: str + The exception's message. + filename: str, optional + The filename for the source code generating this error. + lineno: int, optional + The line number of the error. + colno: int, optional + The column number of the error. + source: str, optional + The actual source code generating this error. + """ + self.message = message + self.filename = filename + self.lineno = lineno + self.colno = colno + self.source = source + super(HySyntaxError, self).__init__(message, + # The builtin `SyntaxError` needs a + # tuple. + (filename, lineno, colno, source)) + + @staticmethod + def from_expression(message, expression, filename=None, source=None): + if not source: + # Maybe the expression object has its own source. + source = getattr(expression, 'source', None) + + if not filename: + filename = getattr(expression, 'filename', None) + + if source: + lineno = expression.start_line + colno = expression.start_column + end_line = getattr(expression, 'end_line', len(source)) + lines = source.splitlines() + source = '\n'.join(lines[lineno-1:end_line]) + else: + # We could attempt to extract the source given a filename, but we + # don't. + lineno = colno = -1 + + return HySyntaxError(message, filename, lineno, colno, source) + + def __str__(self): + + output = traceback.format_exception_only(SyntaxError, self) + + output[-1] = colored.yellow(output[-1]) + if len(self.source) > 0: + output[-2] = colored.green(output[-2]) + for line in output[::-2]: + if line.strip().startswith( + 'File "{}", line'.format(self.filename)): + break + output[-3] = colored.red(output[-3]) + + # Avoid "...expected str instance, ColoredString found" + return reduce(lambda x, y: x + y, output) diff --git a/hy/importer.py b/hy/importer.py index e3cc50d..0e59498 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -17,10 +17,8 @@ import importlib from functools import partial from contextlib import contextmanager -from hy.errors import HyTypeError from hy.compiler import hy_compile, hy_ast_compile_flags from hy.lex import hy_parse -from hy.lex.exceptions import LexException from hy._compat import PY3 @@ -153,15 +151,9 @@ if PY3: def _hy_source_to_code(self, data, path, _optimize=-1): if _could_be_hy_src(path): source = data.decode("utf-8") - try: - hy_tree = hy_parse(source) - with loader_module_obj(self) as module: - data = hy_compile(hy_tree, module) - except (HyTypeError, LexException) as e: - if e.source is None: - e.source = source - e.filename = path - raise + hy_tree = hy_parse(source, filename=path) + with loader_module_obj(self) as module: + data = hy_compile(hy_tree, module) return _py_source_to_code(self, data, path, _optimize=_optimize) @@ -287,19 +279,15 @@ else: fullname = self._fix_name(fullname) if fullname is None: fullname = self.fullname - try: - hy_source = self.get_source(fullname) - hy_tree = hy_parse(hy_source) - with loader_module_obj(self) as module: - hy_ast = hy_compile(hy_tree, module) - code = compile(hy_ast, self.filename, 'exec', - hy_ast_compile_flags) - except (HyTypeError, LexException) as e: - if e.source is None: - e.source = hy_source - e.filename = self.filename - raise + hy_source = self.get_source(fullname) + hy_tree = hy_parse(hy_source, filename=self.filename) + + with loader_module_obj(self) as module: + hy_ast = hy_compile(hy_tree, module) + + code = compile(hy_ast, self.filename, 'exec', + hy_ast_compile_flags) if not sys.dont_write_bytecode: try: @@ -453,7 +441,7 @@ else: try: flags = None if _could_be_hy_src(filename): - hy_tree = hy_parse(source_str) + hy_tree = hy_parse(source_str, filename=filename) if module is None: module = inspect.getmodule(inspect.stack()[1][0]) @@ -465,9 +453,6 @@ else: codeobject = compile(source, dfile or filename, 'exec', flags) except Exception as err: - if isinstance(err, (HyTypeError, LexException)) and err.source is None: - err.source = source_str - err.filename = filename py_exc = py_compile.PyCompileError(err.__class__, err, dfile or filename) diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py index bf82cf9..f1465cd 100644 --- a/hy/lex/__init__.py +++ b/hy/lex/__init__.py @@ -8,9 +8,10 @@ import re import sys import unicodedata -from hy._compat import str_type, isidentifier, UCS4 +from hy._compat import str_type, isidentifier, UCS4, reraise from hy.lex.exceptions import PrematureEndOfInput, LexException # NOQA from hy.models import HyExpression, HySymbol +from hy.errors import HySyntaxError try: from io import StringIO @@ -18,7 +19,7 @@ except ImportError: from StringIO import StringIO -def hy_parse(source): +def hy_parse(source, filename=''): """Parse a Hy source string. Parameters @@ -26,31 +27,51 @@ def hy_parse(source): source: string Source code to parse. + filename: string, optional + File name corresponding to source. Defaults to "". + Returns ------- - out : instance of `types.CodeType` + out : HyExpression """ - source = re.sub(r'\A#!.*', '', source) - return HyExpression([HySymbol("do")] + tokenize(source + "\n")) + _source = re.sub(r'\A#!.*', '', source) + try: + res = HyExpression([HySymbol("do")] + + tokenize(_source + "\n", + filename=filename)) + res.source = source + res.filename = filename + return res + except HySyntaxError as e: + reraise(type(e), e, None) -def tokenize(buf): - """ - Tokenize a Lisp file or string buffer into internal Hy objects. +class ParserState(object): + def __init__(self, source, filename): + self.source = source + self.filename = filename + + +def tokenize(source, filename=None): + """ Tokenize a Lisp file or string buffer into internal Hy objects. + + Parameters + ---------- + source: str + The source to tokenize. + filename: str, optional + The filename corresponding to `source`. """ from hy.lex.lexer import lexer from hy.lex.parser import parser from rply.errors import LexingError try: - return parser.parse(lexer.lex(buf)) + return parser.parse(lexer.lex(source), + state=ParserState(source, filename)) except LexingError as e: pos = e.getsourcepos() - raise LexException("Could not identify the next token.", - pos.lineno, pos.colno, buf) - except LexException as e: - if e.source is None: - e.source = buf - raise + raise LexException("Could not identify the next token.", filename, + pos.lineno, pos.colno, source) mangle_delim = 'X' diff --git a/hy/lex/exceptions.py b/hy/lex/exceptions.py index 573a8e8..fb9aa7a 100644 --- a/hy/lex/exceptions.py +++ b/hy/lex/exceptions.py @@ -1,49 +1,34 @@ # Copyright 2019 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. - -from hy.errors import HyError +from hy.errors import HySyntaxError -class LexException(HyError): - """Error during the Lexing of a Hython expression.""" - def __init__(self, message, lineno, colno, source=None): - super(LexException, self).__init__(message) - self.message = message - self.lineno = lineno - self.colno = colno - self.source = source - self.filename = '' +class LexException(HySyntaxError): - def __str__(self): - from hy.errors import colored + @classmethod + def from_lexer(cls, message, state, token): + source_pos = token.getsourcepos() + if token.source_pos: + lineno = source_pos.lineno + colno = source_pos.colno + else: + lineno = -1 + colno = -1 - line = self.lineno - start = self.colno + if state.source: + lines = state.source.splitlines() + if lines[-1] == '': + del lines[-1] - result = "" + if lineno < 1: + lineno = len(lines) + if colno < 1: + colno = len(lines[-1]) - source = self.source.split("\n") - - if line > 0 and start > 0: - result += ' File "%s", line %d, column %d\n\n' % (self.filename, - line, - start) - - if len(self.source) > 0: - source_line = source[line-1] - else: - source_line = "" - - result += ' %s\n' % colored.red(source_line) - result += ' %s%s\n' % (' '*(start-1), colored.green('^')) - - result += colored.yellow("LexException: %s\n\n" % self.message) - - return result + source = lines[lineno - 1] + return cls(message, state.filename, lineno, colno, source) class PrematureEndOfInput(LexException): - """We got a premature end of input""" - def __init__(self, message): - super(PrematureEndOfInput, self).__init__(message, -1, -1) + pass diff --git a/hy/lex/parser.py b/hy/lex/parser.py index c602734..f5cd5e5 100755 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -22,10 +22,10 @@ pg = ParserGenerator([rule.name for rule in lexer.rules] + ['$end']) def set_boundaries(fun): @wraps(fun) - def wrapped(p): + def wrapped(state, p): start = p[0].source_pos end = p[-1].source_pos - ret = fun(p) + ret = fun(state, p) ret.start_line = start.lineno ret.start_column = start.colno if start is not end: @@ -40,9 +40,9 @@ def set_boundaries(fun): def set_quote_boundaries(fun): @wraps(fun) - def wrapped(p): + def wrapped(state, p): start = p[0].source_pos - ret = fun(p) + ret = fun(state, p) ret.start_line = start.lineno ret.start_column = start.colno ret.end_line = p[-1].end_line @@ -52,54 +52,45 @@ def set_quote_boundaries(fun): @pg.production("main : list_contents") -def main(p): +def main(state, p): return p[0] @pg.production("main : $end") -def main_empty(p): +def main_empty(state, p): return [] -def reject_spurious_dots(*items): - "Reject the spurious dots from items" - for list in items: - for tok in list: - if tok == "." and type(tok) == HySymbol: - raise LexException("Malformed dotted list", - tok.start_line, tok.start_column) - - @pg.production("paren : LPAREN list_contents RPAREN") @set_boundaries -def paren(p): +def paren(state, p): return HyExpression(p[1]) @pg.production("paren : LPAREN RPAREN") @set_boundaries -def empty_paren(p): +def empty_paren(state, p): return HyExpression([]) @pg.production("list_contents : term list_contents") -def list_contents(p): +def list_contents(state, p): return [p[0]] + p[1] @pg.production("list_contents : term") -def list_contents_single(p): +def list_contents_single(state, p): return [p[0]] @pg.production("list_contents : DISCARD term discarded_list_contents") -def list_contents_empty(p): +def list_contents_empty(state, p): return [] @pg.production("discarded_list_contents : DISCARD term discarded_list_contents") @pg.production("discarded_list_contents :") -def discarded_list_contents(p): +def discarded_list_contents(state, p): pass @@ -109,58 +100,58 @@ def discarded_list_contents(p): @pg.production("term : list") @pg.production("term : set") @pg.production("term : string") -def term(p): +def term(state, p): return p[0] @pg.production("term : DISCARD term term") -def term_discard(p): +def term_discard(state, p): return p[2] @pg.production("term : QUOTE term") @set_quote_boundaries -def term_quote(p): +def term_quote(state, p): return HyExpression([HySymbol("quote"), p[1]]) @pg.production("term : QUASIQUOTE term") @set_quote_boundaries -def term_quasiquote(p): +def term_quasiquote(state, p): return HyExpression([HySymbol("quasiquote"), p[1]]) @pg.production("term : UNQUOTE term") @set_quote_boundaries -def term_unquote(p): +def term_unquote(state, p): return HyExpression([HySymbol("unquote"), p[1]]) @pg.production("term : UNQUOTESPLICE term") @set_quote_boundaries -def term_unquote_splice(p): +def term_unquote_splice(state, p): return HyExpression([HySymbol("unquote-splice"), p[1]]) @pg.production("term : HASHSTARS term") @set_quote_boundaries -def term_hashstars(p): +def term_hashstars(state, p): n_stars = len(p[0].getstr()[1:]) if n_stars == 1: sym = "unpack-iterable" elif n_stars == 2: sym = "unpack-mapping" else: - raise LexException( + raise LexException.from_lexer( "Too many stars in `#*` construct (if you want to unpack a symbol " "beginning with a star, separate it with whitespace)", - p[0].source_pos.lineno, p[0].source_pos.colno) + state, p[0]) return HyExpression([HySymbol(sym), p[1]]) @pg.production("term : HASHOTHER term") @set_quote_boundaries -def hash_other(p): +def hash_other(state, p): # p == [(Token('HASHOTHER', '#foo'), bar)] st = p[0].getstr()[1:] str_object = HyString(st) @@ -170,63 +161,63 @@ def hash_other(p): @pg.production("set : HLCURLY list_contents RCURLY") @set_boundaries -def t_set(p): +def t_set(state, p): return HySet(p[1]) @pg.production("set : HLCURLY RCURLY") @set_boundaries -def empty_set(p): +def empty_set(state, p): return HySet([]) @pg.production("dict : LCURLY list_contents RCURLY") @set_boundaries -def t_dict(p): +def t_dict(state, p): return HyDict(p[1]) @pg.production("dict : LCURLY RCURLY") @set_boundaries -def empty_dict(p): +def empty_dict(state, p): return HyDict([]) @pg.production("list : LBRACKET list_contents RBRACKET") @set_boundaries -def t_list(p): +def t_list(state, p): return HyList(p[1]) @pg.production("list : LBRACKET RBRACKET") @set_boundaries -def t_empty_list(p): +def t_empty_list(state, p): return HyList([]) @pg.production("string : STRING") @set_boundaries -def t_string(p): +def t_string(state, p): # Replace the single double quotes with triple double quotes to allow # embedded newlines. try: s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""') except SyntaxError: - raise LexException("Can't convert {} to a HyString".format(p[0].value), - p[0].source_pos.lineno, p[0].source_pos.colno) + raise LexException.from_lexer("Can't convert {} to a HyString".format(p[0].value), + state, p[0]) return (HyString if isinstance(s, str_type) else HyBytes)(s) @pg.production("string : PARTIAL_STRING") -def t_partial_string(p): +def t_partial_string(state, p): # Any unterminated string requires more input - raise PrematureEndOfInput("Premature end of input") + raise PrematureEndOfInput.from_lexer("Partial string literal", state, p[0]) bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING') @pg.production("string : BRACKETSTRING") @set_boundaries -def t_bracket_string(p): +def t_bracket_string(state, p): m = bracket_string_re.match(p[0].value) delim, content = m.groups() return HyString(content, brackets=delim) @@ -234,7 +225,7 @@ def t_bracket_string(p): @pg.production("identifier : IDENTIFIER") @set_boundaries -def t_identifier(p): +def t_identifier(state, p): obj = p[0].value val = symbol_like(obj) @@ -243,11 +234,11 @@ def t_identifier(p): if "." in obj and symbol_like(obj.split(".", 1)[0]) is not None: # E.g., `5.attr` or `:foo.attr` - raise LexException( + raise LexException.from_lexer( 'Cannot access attribute on anything other than a name (in ' 'order to get attributes of expressions, use ' '`(. )` or `(. )`)', - p[0].source_pos.lineno, p[0].source_pos.colno) + state, p[0]) return HySymbol(obj) @@ -284,14 +275,24 @@ def symbol_like(obj): @pg.error -def error_handler(token): +def error_handler(state, token): tokentype = token.gettokentype() if tokentype == '$end': - raise PrematureEndOfInput("Premature end of input") + source_pos = token.source_pos or token.getsourcepos() + source = state.source + if source_pos: + lineno = source_pos.lineno + colno = source_pos.colno + else: + lineno = -1 + colno = -1 + + raise PrematureEndOfInput.from_lexer("Premature end of input", state, + token) else: - raise LexException( - "Ran into a %s where it wasn't expected." % tokentype, - token.source_pos.lineno, token.source_pos.colno) + raise LexException.from_lexer( + "Ran into a %s where it wasn't expected." % tokentype, state, + token) parser = pg.build() diff --git a/hy/macros.py b/hy/macros.py index be59200..274702f 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -1,14 +1,16 @@ # Copyright 2019 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. +import sys import importlib import inspect import pkgutil -from hy._compat import PY3, string_types +from contextlib import contextmanager + +from hy._compat import PY3, string_types, reraise from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value from hy.lex import mangle - from hy.errors import HyTypeError, HyMacroExpansionError try: @@ -257,6 +259,32 @@ def make_empty_fn_copy(fn): return empty_fn +@contextmanager +def macro_exceptions(module, macro_tree, compiler=None): + try: + yield + except Exception as e: + try: + filename = inspect.getsourcefile(module) + source = inspect.getsource(module) + except TypeError: + if compiler: + filename = compiler.filename + source = compiler.source + + if not isinstance(e, HyTypeError): + exc_type = HyMacroExpansionError + msg = "expanding `{}': ".format(macro_tree[0]) + msg += str(e).replace("()", "", 1).strip() + else: + exc_type = HyTypeError + msg = e.message + + reraise(exc_type, + exc_type(msg, filename, macro_tree, source), + sys.exc_info()[2].tb_next) + + def macroexpand(tree, module, compiler=None, once=False): """Expand the toplevel macros for the given Hy AST tree. @@ -324,23 +352,10 @@ def macroexpand(tree, module, compiler=None, once=False): compiler = HyASTCompiler(module) opts['compiler'] = compiler - try: + with macro_exceptions(module, tree, compiler): m_copy = make_empty_fn_copy(m) m_copy(module.__name__, *tree[1:], **opts) - except TypeError as e: - msg = "expanding `" + str(tree[0]) + "': " - msg += str(e).replace("()", "", 1).strip() - raise HyMacroExpansionError(tree, msg) - - try: obj = m(module.__name__, *tree[1:], **opts) - except HyTypeError as e: - if e.expression is None: - e.expression = tree - raise - except Exception as e: - msg = "expanding `" + str(tree[0]) + "': " + repr(e) - raise HyMacroExpansionError(tree, msg) if isinstance(obj, HyExpression): obj.module = inspect.getmodule(m) @@ -375,7 +390,8 @@ def tag_macroexpand(tag, tree, module): None) if tag_macro is None: - raise HyTypeError(tag, "'{0}' is not a defined tag macro.".format(tag)) + raise HyTypeError("`{0}' is not a defined tag macro.".format(tag), + None, tag, None) expr = tag_macro(tree) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 75e9c49..0c7bcd5 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -10,7 +10,7 @@ from hy.models import HyObject from hy.compiler import hy_compile, hy_eval from hy.errors import HyCompileError, HyTypeError from hy.lex import hy_parse -from hy.lex.exceptions import LexException +from hy.lex.exceptions import LexException, PrematureEndOfInput from hy._compat import PY3 import ast @@ -474,7 +474,7 @@ def test_lambda_list_keywords_kwonly(): else: exception = cant_compile(kwonly_demo) assert isinstance(exception, HyTypeError) - message, = exception.args + message = exception.args[0] assert message == "&kwonly parameters require Python 3" @@ -547,7 +547,7 @@ def test_compile_error(): def test_for_compile_error(): """Ensure we get compile error in tricky 'for' cases""" - with pytest.raises(LexException) as excinfo: + with pytest.raises(PrematureEndOfInput) as excinfo: can_compile("(fn [] (for)") assert excinfo.value.message == "Premature end of input" diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 3017d16..dea1baf 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -14,10 +14,10 @@ from fractions import Fraction import pytest import hy -from hy.errors import HyTypeError from hy.lex import hy_parse -from hy.lex.exceptions import LexException -from hy.compiler import hy_compile +from hy.errors import HyLanguageError +from hy.lex.exceptions import PrematureEndOfInput +from hy.compiler import hy_eval, hy_compile from hy.importer import HyLoader, cache_from_source try: @@ -57,7 +57,7 @@ def test_runpy(): def test_stringer(): - _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '__main__') + _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), __name__) assert type(_ast.body[0]) == ast.FunctionDef @@ -79,14 +79,8 @@ def test_imports(): def test_import_error_reporting(): "Make sure that (import) reports errors correctly." - def _import_error_test(): - try: - _ = hy_compile(hy_parse("(import \"sys\")"), '__main__') - except HyTypeError: - return "Error reported" - - assert _import_error_test() == "Error reported" - assert _import_error_test() is not None + with pytest.raises(HyLanguageError): + hy_compile(hy_parse("(import \"sys\")"), __name__) def test_import_error_cleanup(): @@ -124,7 +118,7 @@ def test_import_autocompiles(): def test_eval(): def eval_str(s): - return hy.eval(hy.read_str(s)) + return hy_eval(hy.read_str(s), filename='', source=s) assert eval_str('[1 2 3]') == [1, 2, 3] assert eval_str('{"dog" "bark" "cat" "meow"}') == { @@ -205,8 +199,7 @@ def test_reload(): assert mod.a == 11 assert mod.b == 20 - # Now cause a `LexException`, and confirm that the good module and its - # contents stick around. + # Now cause a syntax error unlink(source) with open(source, "w") as f: @@ -214,7 +207,7 @@ def test_reload(): f.write("(setv a 11") f.write("(setv b (// 20 1))") - with pytest.raises(LexException): + with pytest.raises(PrematureEndOfInput): reload(mod) mod = sys.modules.get(TESTFN) diff --git a/tests/test_bin.py b/tests/test_bin.py index 6336122..8aef923 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -149,7 +149,7 @@ def test_bin_hy_stdin_unlocatable_hytypeerror(): # inside run_cmd. _, err = run_cmd("hy", """ (import hy.errors) - (raise (hy.errors.HyTypeError '[] (+ "A" "Z")))""") + (raise (hy.errors.HyTypeError (+ "A" "Z") None '[] None))""") assert "AZ" in err diff --git a/tests/test_lex.py b/tests/test_lex.py index 19da88b..c759624 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -1,13 +1,15 @@ # Copyright 2019 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. +import traceback + +import pytest from math import isnan from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol, HyString, HyDict, HyList, HySet, HyKeyword) from hy.lex import tokenize from hy.lex.exceptions import LexException, PrematureEndOfInput -import pytest def peoi(): return pytest.raises(PrematureEndOfInput) def lexe(): return pytest.raises(LexException) @@ -180,7 +182,23 @@ def test_lex_digit_separators(): def test_lex_bad_attrs(): - with lexe(): tokenize("1.foo") + with lexe() as execinfo: + tokenize("1.foo") + + expected = [ + ' File "", line 1\n', + ' 1.foo\n', + ' ^\n', + ('LexException: Cannot access attribute on anything other' + ' than a name (in order to get attributes of expressions,' + ' use `(. )` or `(. )`)\n') + ] + output = traceback.format_exception_only(execinfo.type, execinfo.value) + + assert output[:-1:1] == expected[:-1:1] + # Python 2.7 doesn't give the full exception name, so we compensate. + assert output[-1].endswith(expected[-1]) + with lexe(): tokenize("0.foo") with lexe(): tokenize("1.5.foo") with lexe(): tokenize("1e3.foo") From e468d5f081af576a3321a0d4a5007ed5422095c0 Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Sun, 28 Oct 2018 21:42:18 -0500 Subject: [PATCH 3/8] Refactor REPL error handling and filter Hy internal trace output These changes make the Hy REPL more closely follow `code.InteractiveConsole`'s class interface and provide minimally intrusive traceback print-out filtering via a context manager that temporarily alters `sys.excepthook`. In other words, exception messages from the REPL will no longer show Hy internal code (e.g. importer, compiler and parsing functions). The boolean variable `hy.errors._hy_filter_internal_errors` dynamically enables/disables trace filtering, and the env variable `HY_FILTER_INTERNAL_ERRORS` can be used as the initial value. --- docs/language/cli.rst | 6 -- hy/__init__.py | 10 ++++ hy/cmdline.py | 125 +++++++++++++++++++++--------------------- hy/compiler.py | 2 +- hy/errors.py | 82 ++++++++++++++++++++++++++- tests/test_lex.py | 80 ++++++++++++++++++++++----- 6 files changed, 218 insertions(+), 87 deletions(-) diff --git a/docs/language/cli.rst b/docs/language/cli.rst index 2c8a1f7..e59640d 100644 --- a/docs/language/cli.rst +++ b/docs/language/cli.rst @@ -48,12 +48,6 @@ Command Line Options `--spy` only works on REPL mode. .. versionadded:: 0.9.11 -.. cmdoption:: --show-tracebacks - - Print extended tracebacks for Hy exceptions. - - .. versionadded:: 0.9.12 - .. cmdoption:: --repl-output-fn Format REPL output using specific function (e.g., hy.contrib.hy-repr.hy-repr) diff --git a/hy/__init__.py b/hy/__init__.py index f188b64..eb1d91c 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -5,6 +5,16 @@ except ImportError: __version__ = 'unknown' +def _initialize_env_var(env_var, default_val): + import os, distutils.util + try: + res = bool(distutils.util.strtobool( + os.environ.get(env_var, str(default_val)))) + except ValueError as e: + res = default_val + return res + + from hy.models import HyExpression, HyInteger, HyKeyword, HyComplex, HyString, HyBytes, HySymbol, HyFloat, HyDict, HyList, HySet # NOQA diff --git a/hy/cmdline.py b/hy/cmdline.py index 70d7b72..f7cca52 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -21,7 +21,7 @@ import hy from hy.lex import hy_parse, mangle from hy.lex.exceptions import PrematureEndOfInput from hy.compiler import HyASTCompiler, hy_compile, hy_eval -from hy.errors import HyTypeError, HyLanguageError, HySyntaxError +from hy.errors import HySyntaxError, filtered_hy_exceptions from hy.importer import runhy from hy.completer import completion, Completer from hy.macros import macro, require @@ -90,47 +90,58 @@ class HyREPL(code.InteractiveConsole, object): self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] self.locals.update({sym: None for sym in self._repl_results_symbols}) - def runsource(self, source, filename='', symbol='single'): - global SIMPLE_TRACEBACKS + def ast_callback(self, main_ast, expr_ast): + if self.spy: + # Mush the two AST chunks into a single module for + # conversion into Python. + new_ast = ast.Module(main_ast.body + + [ast.Expr(expr_ast.body)]) + print(astor.to_source(new_ast)) - def error_handler(e, use_simple_traceback=False): - self.locals[mangle("*e")] = e - if use_simple_traceback: - print(e, file=sys.stderr) - else: - self.showtraceback() + def _error_wrap(self, error_fn, *args, **kwargs): + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + + # Sadly, this method in Python 2.7 ignores an overridden + # `sys.excepthook`. + if sys.excepthook is sys.__excepthook__: + error_fn(*args, **kwargs) + else: + sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback) + + self.locals[mangle("*e")] = sys.last_value + + def showsyntaxerror(self, filename=None): + if filename is None: + filename = self.filename + + self._error_wrap(super(HyREPL, self).showsyntaxerror, + filename=filename) + + def showtraceback(self): + self._error_wrap(super(HyREPL, self).showtraceback) + + def runsource(self, source, filename='', symbol='single'): try: do = hy_parse(source, filename=filename) except PrematureEndOfInput: return True except HySyntaxError as e: - error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS) + self.showsyntaxerror(filename=filename) return False try: - def ast_callback(main_ast, expr_ast): - if self.spy: - # Mush the two AST chunks into a single module for - # conversion into Python. - new_ast = ast.Module(main_ast.body + - [ast.Expr(expr_ast.body)]) - print(astor.to_source(new_ast)) - - value = hy_eval(do, self.locals, self.module, - ast_callback=ast_callback, - compiler=self.hy_compiler, - filename=filename, + # Our compiler doesn't correspond to a real, fixed source file, so + # we need to [re]set these. + self.hy_compiler.filename = filename + self.hy_compiler.source = source + value = hy_eval(do, self.locals, self.module, self.ast_callback, + compiler=self.hy_compiler, filename=filename, source=source) - - except HyTypeError as e: - if e.source is None: - e.source = source - e.filename = filename - error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS) - return False + except SystemExit: + raise except Exception as e: - error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS) + self.showtraceback() return False if value is not None: @@ -142,10 +153,12 @@ class HyREPL(code.InteractiveConsole, object): # Print the value. try: output = self.output_fn(value) - except Exception as e: - error_handler(e) + except Exception: + self.showtraceback() return False + print(output) + return False @@ -201,25 +214,12 @@ def ideas_macro(ETname): """)]) -SIMPLE_TRACEBACKS = True - - -def pretty_error(func, *args, **kw): - try: - return func(*args, **kw) - except HyLanguageError as e: - if SIMPLE_TRACEBACKS: - print(e, file=sys.stderr) - sys.exit(1) - raise - - def run_command(source, filename=None): tree = hy_parse(source, filename=filename) __main__ = importlib.import_module('__main__') require("hy.cmdline", __main__, assignments="ALL") - pretty_error(hy_eval, tree, None, __main__, filename=filename, - source=source) + with filtered_hy_exceptions(): + hy_eval(tree, None, __main__, filename=filename, source=source) return 0 @@ -232,9 +232,7 @@ def run_repl(hr=None, **kwargs): hr = HyREPL(**kwargs) namespace = hr.locals - - with completion(Completer(namespace)): - + with filtered_hy_exceptions(), completion(Completer(namespace)): hr.interact("{appname} {version} using " "{py}({build}) {pyversion} on {os}".format( appname=hy.__appname__, @@ -263,9 +261,10 @@ def run_icommand(source, **kwargs): else: filename = '' - hr = HyREPL(**kwargs) - hr.runsource(source, filename=filename, symbol='single') - return run_repl(hr) + with filtered_hy_exceptions(): + hr = HyREPL(**kwargs) + hr.runsource(source, filename=filename, symbol='single') + return run_repl(hr) USAGE = "%(prog)s [-h | -i cmd | -c cmd | -m module | file | -] [arg] ..." @@ -301,9 +300,6 @@ def cmdline_handler(scriptname, argv): "(e.g., hy.contrib.hy-repr.hy-repr)") parser.add_argument("-v", "--version", action="version", version=VERSION) - parser.add_argument("--show-tracebacks", action="store_true", - help="show complete tracebacks for Hy exceptions") - # this will contain the script/program name and any arguments for it. parser.add_argument('args', nargs=argparse.REMAINDER, help=argparse.SUPPRESS) @@ -328,10 +324,6 @@ def cmdline_handler(scriptname, argv): options = parser.parse_args(argv[1:]) - if options.show_tracebacks: - global SIMPLE_TRACEBACKS - SIMPLE_TRACEBACKS = False - if options.E: # User did "hy -E ..." _remove_python_envs() @@ -372,7 +364,8 @@ def cmdline_handler(scriptname, argv): try: sys.argv = options.args - runhy.run_path(filename, run_name='__main__') + with filtered_hy_exceptions(): + runhy.run_path(filename, run_name='__main__') return 0 except FileNotFoundError as e: print("hy: Can't open file '{0}': [Errno {1}] {2}".format( @@ -448,9 +441,11 @@ def hy2py_main(): if options.FILE is None or options.FILE == '-': source = sys.stdin.read() - hst = pretty_error(hy_parse, source, filename='') + with filtered_hy_exceptions(): + hst = hy_parse(source, filename='') else: - with io.open(options.FILE, 'r', encoding='utf-8') as source_file: + with filtered_hy_exceptions(), \ + io.open(options.FILE, 'r', encoding='utf-8') as source_file: source = source_file.read() hst = hy_parse(source, filename=options.FILE) @@ -468,7 +463,9 @@ def hy2py_main(): print() print() - _ast = pretty_error(hy_compile, hst, '__main__') + with filtered_hy_exceptions(): + _ast = hy_compile(hst, '__main__') + if options.with_ast: if PY3 and platform.system() == "Windows": _print_for_windows(astor.dump_tree(_ast)) diff --git a/hy/compiler.py b/hy/compiler.py index 5377c33..a02e71a 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1834,7 +1834,7 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False): def hy_eval(hytree, locals=None, module=None, ast_callback=None, - compiler=None, filename='', source=None): + compiler=None, filename=None, source=None): """Evaluates a quoted expression and returns the value. If you're evaluating hand-crafted AST trees, make sure the line numbers diff --git a/hy/errors.py b/hy/errors.py index e8e9f98..4ed56e5 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -2,13 +2,20 @@ # Copyright 2019 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. - +import os +import sys import traceback +import pkgutil from functools import reduce +from contextlib import contextmanager +from hy import _initialize_env_var from clint.textui import colored +_hy_filter_internal_errors = _initialize_env_var('HY_FILTER_INTERNAL_ERRORS', + True) + class HyError(Exception): def __init__(self, message, *args): @@ -193,7 +200,8 @@ class HySyntaxError(HyLanguageError, SyntaxError): def __str__(self): - output = traceback.format_exception_only(SyntaxError, self) + output = traceback.format_exception_only(SyntaxError, + SyntaxError(*self.args)) output[-1] = colored.yellow(output[-1]) if len(self.source) > 0: @@ -206,3 +214,73 @@ class HySyntaxError(HyLanguageError, SyntaxError): # Avoid "...expected str instance, ColoredString found" return reduce(lambda x, y: x + y, output) + + +def _get_module_info(module): + compiler_loader = pkgutil.get_loader(module) + is_pkg = compiler_loader.is_package(module) + filename = compiler_loader.get_filename() + if is_pkg: + # Use package directory + return os.path.dirname(filename) + else: + # Normalize filename endings, because tracebacks will use `pyc` when + # the loader says `py`. + return filename.replace('.pyc', '.py') + + +_tb_hidden_modules = {_get_module_info(m) + for m in ['hy.compiler', 'hy.lex', + 'hy.cmdline', 'hy.lex.parser', + 'hy.importer', 'hy._compat', + 'hy.macros', 'hy.models', + 'rply']} + + +def hy_exc_handler(exc_type, exc_value, exc_traceback): + """Produce exceptions print-outs with all frames originating from the + modules in `_tb_hidden_modules` filtered out. + + The frames are actually filtered by each module's filename and only when a + subclass of `HyLanguageError` is emitted. + + This does not remove the frames from the actual tracebacks, so debugging + will show everything. + """ + try: + # frame = (filename, line number, function name*, text) + new_tb = [frame for frame in traceback.extract_tb(exc_traceback) + if not (frame[0].replace('.pyc', '.py') in _tb_hidden_modules or + os.path.dirname(frame[0]) in _tb_hidden_modules)] + + lines = traceback.format_list(new_tb) + + if lines: + lines.insert(0, "Traceback (most recent call last):\n") + + lines.extend(traceback.format_exception_only(exc_type, exc_value)) + output = ''.join(lines) + + sys.stderr.write(output) + sys.stderr.flush() + except Exception: + sys.__excepthook__(exc_type, exc_value, exc_traceback) + + +@contextmanager +def filtered_hy_exceptions(): + """Temporarily apply a `sys.excepthook` that filters Hy internal frames + from tracebacks. + + Filtering can be controlled by the variable + `hy.errors._hy_filter_internal_errors` and environment variable + `HY_FILTER_INTERNAL_ERRORS`. + """ + global _hy_filter_internal_errors + if _hy_filter_internal_errors: + current_hook = sys.excepthook + sys.excepthook = hy_exc_handler + yield + sys.excepthook = current_hook + else: + yield diff --git a/tests/test_lex.py b/tests/test_lex.py index c759624..b0a03dc 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -1,6 +1,7 @@ # Copyright 2019 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. +import sys import traceback import pytest @@ -10,11 +11,36 @@ from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol, HyString, HyDict, HyList, HySet, HyKeyword) from hy.lex import tokenize from hy.lex.exceptions import LexException, PrematureEndOfInput +from hy.errors import hy_exc_handler def peoi(): return pytest.raises(PrematureEndOfInput) def lexe(): return pytest.raises(LexException) +def check_ex(execinfo, expected): + output = traceback.format_exception_only(execinfo.type, execinfo.value) + assert output[:-1] == expected[:-1] + # Python 2.7 doesn't give the full exception name, so we compensate. + assert output[-1].endswith(expected[-1]) + + +def check_trace_output(capsys, execinfo, expected): + sys.__excepthook__(execinfo.type, execinfo.value, execinfo.tb) + captured_wo_filtering = capsys.readouterr()[-1].strip('\n') + + hy_exc_handler(execinfo.type, execinfo.value, execinfo.tb) + captured_w_filtering = capsys.readouterr()[-1].strip('\n') + + output = captured_w_filtering.split('\n') + + # Make sure the filtered frames aren't the same as the unfiltered ones. + assert output[:-1] != captured_wo_filtering.split('\n')[:-1] + # Remove the origin frame lines. + assert output[3:-1] == expected[:-1] + # Python 2.7 doesn't give the full exception name, so we compensate. + assert output[-1].endswith(expected[-1]) + + def test_lex_exception(): """ Ensure tokenize throws a fit on a partial input """ with peoi(): tokenize("(foo") @@ -32,8 +58,13 @@ def test_unbalanced_exception(): def test_lex_single_quote_err(): "Ensure tokenizing \"' \" throws a LexException that can be stringified" # https://github.com/hylang/hy/issues/1252 - with lexe() as e: tokenize("' ") - assert "Could not identify the next token" in str(e.value) + with lexe() as execinfo: + tokenize("' ") + check_ex(execinfo, [ + ' File "", line -1\n', + " '\n", + ' ^\n', + 'LexException: Could not identify the next token.\n']) def test_lex_expression_symbols(): @@ -76,7 +107,11 @@ def test_lex_strings_exception(): """ Make sure tokenize throws when codec can't decode some bytes""" with lexe() as execinfo: tokenize('\"\\x8\"') - assert "Can't convert \"\\x8\" to a HyString" in str(execinfo.value) + check_ex(execinfo, [ + ' File "", line 1\n', + ' "\\x8"\n', + ' ^\n', + 'LexException: Can\'t convert "\\x8" to a HyString\n']) def test_lex_bracket_strings(): @@ -184,20 +219,13 @@ def test_lex_digit_separators(): def test_lex_bad_attrs(): with lexe() as execinfo: tokenize("1.foo") - - expected = [ + check_ex(execinfo, [ ' File "", line 1\n', ' 1.foo\n', ' ^\n', - ('LexException: Cannot access attribute on anything other' - ' than a name (in order to get attributes of expressions,' - ' use `(. )` or `(. )`)\n') - ] - output = traceback.format_exception_only(execinfo.type, execinfo.value) - - assert output[:-1:1] == expected[:-1:1] - # Python 2.7 doesn't give the full exception name, so we compensate. - assert output[-1].endswith(expected[-1]) + 'LexException: Cannot access attribute on anything other' + ' than a name (in order to get attributes of expressions,' + ' use `(. )` or `(. )`)\n']) with lexe(): tokenize("0.foo") with lexe(): tokenize("1.5.foo") @@ -437,3 +465,27 @@ def test_discard(): assert tokenize("a '#_b c") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("c")])] assert tokenize("a '#_b #_c d") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("d")])] assert tokenize("a '#_ #_b c d") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("d")])] + + +def test_lex_exception_filtering(capsys): + """Confirm that the exception filtering works for lexer errors.""" + + # First, test for PrematureEndOfInput + with peoi() as execinfo: + tokenize(" \n (foo") + check_trace_output(capsys, execinfo, [ + ' File "", line 2', + ' (foo', + ' ^', + 'PrematureEndOfInput: Premature end of input']) + + # Now, for a generic LexException + with lexe() as execinfo: + tokenize(" \n\n 1.foo ") + check_trace_output(capsys, execinfo, [ + ' File "", line 3', + ' 1.foo', + ' ^', + 'LexException: Cannot access attribute on anything other' + ' than a name (in order to get attributes of expressions,' + ' use `(. )` or `(. )`)']) From cadfa4152bca314e2e614baa19e2ccbf07ade00e Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Sun, 28 Oct 2018 22:02:08 -0500 Subject: [PATCH 4/8] Make colored output configurable Colored exception output is now disabled by default and configurable through `hy.errors._hy_colored_errors` and the environment variable `HY_COLORED_ERRORS`. Likewise, Hy model/AST color printing is now configurable and disabled by default. The corresponding variables are `hy.models._hy_colored_ast_objects` and `HY_COLORED_AST_OBJECTS`. Closes hylang/hy#1429, closes hylang/hy#1510. --- hy/errors.py | 49 ++++++++++++++++++++++++++++++------------------- hy/models.py | 12 ++++++++---- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/hy/errors.py b/hy/errors.py index 4ed56e5..9ca823e 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -15,6 +15,7 @@ from clint.textui import colored _hy_filter_internal_errors = _initialize_env_var('HY_FILTER_INTERNAL_ERRORS', True) +_hy_colored_errors = _initialize_env_var('HY_COLORED_ERRORS', False) class HyError(Exception): @@ -73,9 +74,16 @@ class HyTypeError(HyLanguageError, TypeError): source) def __str__(self): + global _hy_colored_errors result = "" + if _hy_colored_errors: + from clint.textui import colored + red, green, yellow = colored.red, colored.green, colored.yellow + else: + red = green = yellow = lambda x: x + if all(getattr(self.expression, x, None) is not None for x in ("start_line", "start_column", "end_column")): @@ -97,28 +105,28 @@ class HyTypeError(HyLanguageError, TypeError): start) if len(source) == 1: - result += ' %s\n' % colored.red(source[0]) + result += ' %s\n' % red(source[0]) result += ' %s%s\n' % (' '*(start-1), - colored.green('^' + '-'*(length-1) + '^')) + green('^' + '-'*(length-1) + '^')) if len(source) > 1: - result += ' %s\n' % colored.red(source[0]) + result += ' %s\n' % red(source[0]) result += ' %s%s\n' % (' '*(start-1), - colored.green('^' + '-'*length)) + green('^' + '-'*length)) if len(source) > 2: # write the middle lines for line in source[1:-1]: - result += ' %s\n' % colored.red("".join(line)) - result += ' %s\n' % colored.green("-"*len(line)) + result += ' %s\n' % red("".join(line)) + result += ' %s\n' % green("-"*len(line)) # write the last line - result += ' %s\n' % colored.red("".join(source[-1])) - result += ' %s\n' % colored.green('-'*(end-1) + '^') + result += ' %s\n' % red("".join(source[-1])) + result += ' %s\n' % green('-'*(end-1) + '^') else: result += ' File "%s", unknown location\n' % self.filename - result += colored.yellow("%s: %s\n\n" % - (self.__class__.__name__, - self.message)) + result += yellow("%s: %s\n\n" % + (self.__class__.__name__, + self.message)) return result @@ -199,18 +207,21 @@ class HySyntaxError(HyLanguageError, SyntaxError): return HySyntaxError(message, filename, lineno, colno, source) def __str__(self): + global _hy_colored_errors output = traceback.format_exception_only(SyntaxError, SyntaxError(*self.args)) - output[-1] = colored.yellow(output[-1]) - if len(self.source) > 0: - output[-2] = colored.green(output[-2]) - for line in output[::-2]: - if line.strip().startswith( - 'File "{}", line'.format(self.filename)): - break - output[-3] = colored.red(output[-3]) + if _hy_colored_errors: + from hy.errors import colored + output[-1] = colored.yellow(output[-1]) + if len(self.source) > 0: + output[-2] = colored.green(output[-2]) + for line in output[::-2]: + if line.strip().startswith( + 'File "{}", line'.format(self.filename)): + break + output[-3] = colored.red(output[-3]) # Avoid "...expected str instance, ColoredString found" return reduce(lambda x, y: x + y, output) diff --git a/hy/models.py b/hy/models.py index cf02dab..134e322 100644 --- a/hy/models.py +++ b/hy/models.py @@ -1,16 +1,17 @@ # Copyright 2019 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. - from __future__ import unicode_literals + from contextlib import contextmanager from math import isnan, isinf +from hy import _initialize_env_var from hy._compat import PY3, str_type, bytes_type, long_type, string_types from fractions import Fraction from clint.textui import colored - PRETTY = True +_hy_colored_ast_objects = _initialize_env_var('HY_COLORED_AST_OBJECTS', False) @contextmanager @@ -271,8 +272,9 @@ class HySequence(HyObject, list): return str(self) if PRETTY else super(HySequence, self).__repr__() def __str__(self): + global _hy_colored_ast_objects with pretty(): - c = self.color + c = self.color if _hy_colored_ast_objects else str if self: return ("{}{}\n {}{}").format( c(self.__class__.__name__), @@ -298,10 +300,12 @@ class HyDict(HySequence): """ HyDict (just a representation of a dict) """ + color = staticmethod(colored.green) def __str__(self): + global _hy_colored_ast_objects with pretty(): - g = colored.green + g = self.color if _hy_colored_ast_objects else str if self: pairs = [] for k, v in zip(self[::2],self[1::2]): From fb6feaf08298d67045583e4dcae090f1e80b2b94 Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Fri, 12 Oct 2018 23:25:43 -0500 Subject: [PATCH 5/8] Improve correspondence with Python errors and console behavior Compiler and command-line error messages now reflect their Python counterparts. E.g. where Python emits a `SyntaxError`, so does Hy; same for `TypeError`s. Multiple tests have been added that check the format and type of raised exceptions over varying command-line invocations (e.g. interactive and not). A new exception type for `require` errors was added so that they can be treated like normal run-time errors and not compiler errors. The Hy REPL has been further refactored to better match the class-structured API. Now, different error types are handled separately and leverage more base class-provided functionality. Closes hylang/hy#1486. --- hy/_compat.py | 24 ++ hy/cmdline.py | 173 +++++++++---- hy/compiler.py | 13 +- hy/core/bootstrap.hy | 10 +- hy/errors.py | 369 ++++++++++++++------------- hy/lex/__init__.py | 26 +- hy/lex/exceptions.py | 35 +-- hy/lex/parser.py | 10 - hy/macros.py | 79 +++--- tests/compilers/test_ast.py | 56 ++-- tests/macros/test_macro_processor.py | 3 +- tests/native_tests/core.hy | 2 +- tests/native_tests/language.hy | 30 +-- tests/native_tests/native_macros.hy | 31 ++- tests/native_tests/operators.hy | 2 +- tests/test_bin.py | 151 ++++++++++- tests/test_lex.py | 4 +- 17 files changed, 630 insertions(+), 388 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index 55180ca..2711445 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -40,6 +40,10 @@ if PY3: finally: traceback = None + code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize', + 'flags', 'code', 'consts', 'names', 'varnames', + 'filename', 'name', 'firstlineno', 'lnotab', 'freevars', + 'cellvars'] else: def raise_from(value, from_value=None): raise value @@ -52,10 +56,30 @@ else: traceback = None ''') + code_obj_args = ['argcount', 'nlocals', 'stacksize', 'flags', 'code', + 'consts', 'names', 'varnames', 'filename', 'name', + 'firstlineno', 'lnotab', 'freevars', 'cellvars'] + raise_code = compile(raise_src, __file__, 'exec') exec(raise_code) +def rename_function(func, new_name): + """Creates a copy of a function and [re]sets the name at the code-object + level. + """ + c = func.__code__ + new_code = type(c)(*[getattr(c, 'co_{}'.format(a)) + if a != 'name' else str(new_name) + for a in code_obj_args]) + + _fn = type(func)(new_code, func.__globals__, str(new_name), + func.__defaults__, func.__closure__) + _fn.__dict__.update(func.__dict__) + + return _fn + + def isidentifier(x): if x in ('True', 'False', 'None', 'print'): # `print` is special-cased here because Python 2's diff --git a/hy/cmdline.py b/hy/cmdline.py index f7cca52..a9f3af3 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -12,6 +12,7 @@ import os import io import importlib import py_compile +import traceback import runpy import types @@ -20,8 +21,9 @@ import astor.code_gen import hy from hy.lex import hy_parse, mangle from hy.lex.exceptions import PrematureEndOfInput -from hy.compiler import HyASTCompiler, hy_compile, hy_eval -from hy.errors import HySyntaxError, filtered_hy_exceptions +from hy.compiler import HyASTCompiler, hy_eval, hy_compile, ast_compile +from hy.errors import (HyLanguageError, HyRequireError, HyMacroExpansionError, + filtered_hy_exceptions, hy_exc_handler) from hy.importer import runhy from hy.completer import completion, Completer from hy.macros import macro, require @@ -50,29 +52,70 @@ builtins.quit = HyQuitter('quit') builtins.exit = HyQuitter('exit') +class HyCommandCompiler(object): + def __init__(self, module, ast_callback=None, hy_compiler=None): + self.module = module + self.ast_callback = ast_callback + self.hy_compiler = hy_compiler + + def __call__(self, source, filename="", symbol="single"): + try: + hy_ast = hy_parse(source, filename=filename) + root_ast = ast.Interactive if symbol == 'single' else ast.Module + + # Our compiler doesn't correspond to a real, fixed source file, so + # we need to [re]set these. + self.hy_compiler.filename = filename + self.hy_compiler.source = source + exec_ast, eval_ast = hy_compile(hy_ast, self.module, root=root_ast, + get_expr=True, + compiler=self.hy_compiler, + filename=filename, source=source) + + if self.ast_callback: + self.ast_callback(exec_ast, eval_ast) + + exec_code = ast_compile(exec_ast, filename, symbol) + eval_code = ast_compile(eval_ast, filename, 'eval') + + return exec_code, eval_code + except PrematureEndOfInput: + # Save these so that we can reraise/display when an incomplete + # interactive command is given at the prompt. + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + return None + + class HyREPL(code.InteractiveConsole, object): def __init__(self, spy=False, output_fn=None, locals=None, - filename=""): - - super(HyREPL, self).__init__(locals=locals, - filename=filename) + filename=""): # Create a proper module for this REPL so that we can obtain it easily # (e.g. using `importlib.import_module`). - # Also, make sure it's properly introduced to `sys.modules` and - # consistently use its namespace as `locals` from here on. + # We let `InteractiveConsole` initialize `self.locals` when it's + # `None`. + super(HyREPL, self).__init__(locals=locals, + filename=filename) + module_name = self.locals.get('__name__', '__console__') + # Make sure our newly created module is properly introduced to + # `sys.modules`, and consistently use its namespace as `self.locals` + # from here on. self.module = sys.modules.setdefault(module_name, types.ModuleType(module_name)) self.module.__dict__.update(self.locals) self.locals = self.module.__dict__ # Load cmdline-specific macros. - require('hy.cmdline', module_name, assignments='ALL') + require('hy.cmdline', self.module, assignments='ALL') self.hy_compiler = HyASTCompiler(self.module) + self.compile = HyCommandCompiler(self.module, self.ast_callback, + self.hy_compiler) + self.spy = spy + self.last_value = None if output_fn is None: self.output_fn = repr @@ -90,13 +133,18 @@ class HyREPL(code.InteractiveConsole, object): self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] self.locals.update({sym: None for sym in self._repl_results_symbols}) - def ast_callback(self, main_ast, expr_ast): + def ast_callback(self, exec_ast, eval_ast): if self.spy: - # Mush the two AST chunks into a single module for - # conversion into Python. - new_ast = ast.Module(main_ast.body + - [ast.Expr(expr_ast.body)]) - print(astor.to_source(new_ast)) + try: + # Mush the two AST chunks into a single module for + # conversion into Python. + new_ast = ast.Module(exec_ast.body + + [ast.Expr(eval_ast.body)]) + print(astor.to_source(new_ast)) + except Exception: + msg = 'Exception in AST callback:\n{}\n'.format( + traceback.format_exc()) + self.write(msg) def _error_wrap(self, error_fn, *args, **kwargs): sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() @@ -120,46 +168,50 @@ class HyREPL(code.InteractiveConsole, object): def showtraceback(self): self._error_wrap(super(HyREPL, self).showtraceback) - def runsource(self, source, filename='', symbol='single'): - + def runcode(self, code): try: - do = hy_parse(source, filename=filename) - except PrematureEndOfInput: - return True - except HySyntaxError as e: - self.showsyntaxerror(filename=filename) - return False - - try: - # Our compiler doesn't correspond to a real, fixed source file, so - # we need to [re]set these. - self.hy_compiler.filename = filename - self.hy_compiler.source = source - value = hy_eval(do, self.locals, self.module, self.ast_callback, - compiler=self.hy_compiler, filename=filename, - source=source) + eval(code[0], self.locals) + self.last_value = eval(code[1], self.locals) + # Don't print `None` values. + self.print_last_value = self.last_value is not None except SystemExit: raise except Exception as e: + # Set this to avoid a print-out of the last value on errors. + self.print_last_value = False + self.showtraceback() + + def runsource(self, source, filename='', symbol='exec'): + try: + res = super(HyREPL, self).runsource(source, filename, symbol) + except (HyMacroExpansionError, HyRequireError): + # We need to handle these exceptions ourselves, because the base + # method only handles `OverflowError`, `SyntaxError` and + # `ValueError`. + self.showsyntaxerror(filename) + return False + except (HyLanguageError): + # Our compiler will also raise `TypeError`s self.showtraceback() return False - if value is not None: - # Shift exisitng REPL results - next_result = value + # Shift exisitng REPL results + if not res: + next_result = self.last_value for sym in self._repl_results_symbols: self.locals[sym], next_result = next_result, self.locals[sym] # Print the value. - try: - output = self.output_fn(value) - except Exception: - self.showtraceback() - return False + if self.print_last_value: + try: + output = self.output_fn(self.last_value) + except Exception: + self.showtraceback() + return False - print(output) + print(output) - return False + return res @macro("koan") @@ -215,9 +267,14 @@ def ideas_macro(ETname): def run_command(source, filename=None): - tree = hy_parse(source, filename=filename) __main__ = importlib.import_module('__main__') require("hy.cmdline", __main__, assignments="ALL") + try: + tree = hy_parse(source, filename=filename) + except HyLanguageError: + hy_exc_handler(*sys.exc_info()) + return 1 + with filtered_hy_exceptions(): hy_eval(tree, None, __main__, filename=filename, source=source) return 0 @@ -259,12 +316,18 @@ def run_icommand(source, **kwargs): source = f.read() filename = source else: - filename = '' + filename = '' + hr = HyREPL(**kwargs) with filtered_hy_exceptions(): - hr = HyREPL(**kwargs) - hr.runsource(source, filename=filename, symbol='single') - return run_repl(hr) + res = hr.runsource(source, filename=filename) + + # If the command was prematurely ended, show an error (just like Python + # does). + if res: + hy_exc_handler(sys.last_type, sys.last_value, sys.last_traceback) + + return run_repl(hr) USAGE = "%(prog)s [-h | -i cmd | -c cmd | -m module | file | -] [arg] ..." @@ -371,6 +434,9 @@ def cmdline_handler(scriptname, argv): print("hy: Can't open file '{0}': [Errno {1}] {2}".format( e.filename, e.errno, e.strerror), file=sys.stderr) sys.exit(e.errno) + except HyLanguageError: + hy_exc_handler(*sys.exc_info()) + sys.exit(1) # User did NOTHING! return run_repl(spy=options.spy, output_fn=options.repl_output_fn) @@ -440,14 +506,15 @@ def hy2py_main(): options = parser.parse_args(sys.argv[1:]) if options.FILE is None or options.FILE == '-': + filename = '' source = sys.stdin.read() - with filtered_hy_exceptions(): - hst = hy_parse(source, filename='') else: - with filtered_hy_exceptions(), \ - io.open(options.FILE, 'r', encoding='utf-8') as source_file: + filename = options.FILE + with io.open(options.FILE, 'r', encoding='utf-8') as source_file: source = source_file.read() - hst = hy_parse(source, filename=options.FILE) + + with filtered_hy_exceptions(): + hst = hy_parse(source, filename=filename) if options.with_source: # need special printing on Windows in case the @@ -464,7 +531,7 @@ def hy2py_main(): print() with filtered_hy_exceptions(): - _ast = hy_compile(hst, '__main__') + _ast = hy_compile(hst, '__main__', filename=filename, source=source) if options.with_ast: if PY3 and platform.system() == "Windows": diff --git a/hy/compiler.py b/hy/compiler.py index a02e71a..08e0c98 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -9,8 +9,8 @@ from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex, from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole, notpexpr, dolike, pexpr, times, Tag, tag, unpack) from funcparserlib.parser import some, many, oneplus, maybe, NoParseError -from hy.errors import (HyCompileError, HyTypeError, HyEvalError, - HyInternalError) +from hy.errors import (HyCompileError, HyTypeError, HyLanguageError, + HySyntaxError, HyEvalError, HyInternalError) from hy.lex import mangle, unmangle @@ -443,15 +443,18 @@ class HyASTCompiler(object): # nested; so let's re-raise this exception, let's not wrap it in # another HyCompileError! raise - except HyTypeError as e: - reraise(type(e), e, None) + except HyLanguageError as e: + # These are expected errors that should be passed to the user. + reraise(type(e), e, sys.exc_info()[2]) except Exception as e: + # These are unexpected errors that will--hopefully--never be seen + # by the user. f_exc = traceback.format_exc() exc_msg = "Internal Compiler Bug 😱\n⤷ {}".format(f_exc) reraise(HyCompileError, HyCompileError(exc_msg), sys.exc_info()[2]) def _syntax_error(self, expr, message): - return HyTypeError(message, self.filename, expr, self.source) + return HySyntaxError(message, expr, self.filename, self.source) def _compile_collect(self, exprs, with_kwargs=False, dict_display=False, oldpy_unpack=False): diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy index 0c98f60..0d3d737 100644 --- a/hy/core/bootstrap.hy +++ b/hy/core/bootstrap.hy @@ -15,13 +15,13 @@ (raise (hy.errors.HyTypeError (% "received a `%s' instead of a symbol for macro name" - (. (type name) --name--)) - --file-- macro-name None))) + (. (type name) __name__)) + None --file-- None))) (for [kw '[&kwonly &kwargs]] (if* (in kw lambda-list) (raise (hy.errors.HyTypeError (% "macros cannot use %s" kw) - --file-- macro-name None)))) + macro-name --file-- None)))) ;; this looks familiar... `(eval-and-compile (import hy) @@ -46,10 +46,10 @@ (raise (hy.errors.HyTypeError (% "received a `%s' instead of a symbol for tag macro name" (. (type tag-name) --name--)) - --file-- tag-name None))) + tag-name --file-- None))) (if (or (= tag-name ":") (= tag-name "&")) - (raise (NameError (% "%s can't be used as a tag macro name" tag-name)))) + (raise (hy.errors.HyNameError (% "%s can't be used as a tag macro name" tag-name)))) (setv tag-name (.replace (hy.models.HyString tag-name) tag-name)) `(eval-and-compile diff --git a/hy/errors.py b/hy/errors.py index 9ca823e..0579e96 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -3,6 +3,7 @@ # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. import os +import re import sys import traceback import pkgutil @@ -19,9 +20,7 @@ _hy_colored_errors = _initialize_env_var('HY_COLORED_ERRORS', False) class HyError(Exception): - def __init__(self, message, *args): - self.message = message - super(HyError, self).__init__(message, *args) + pass class HyInternalError(HyError): @@ -31,9 +30,6 @@ class HyInternalError(HyError): hopefully, never be seen by users! """ - def __init__(self, message, *args): - super(HyInternalError, self).__init__(message, *args) - class HyLanguageError(HyError): """Errors caused by invalid use of the Hy language. @@ -41,8 +37,127 @@ class HyLanguageError(HyError): This, and any errors inheriting from this, are user-facing. """ - def __init__(self, message, *args): - super(HyLanguageError, self).__init__(message, *args) + def __init__(self, message, expression=None, filename=None, source=None, + lineno=1, colno=1): + """ + Parameters + ---------- + message: str + The message to display for this error. + expression: HyObject, optional + The Hy expression generating this error. + filename: str, optional + The filename for the source code generating this error. + Expression-provided information will take precedence of this value. + source: str, optional + The actual source code generating this error. Expression-provided + information will take precedence of this value. + lineno: int, optional + The line number of the error. Expression-provided information will + take precedence of this value. + colno: int, optional + The column number of the error. Expression-provided information + will take precedence of this value. + """ + self.msg = message + self.compute_lineinfo(expression, filename, source, lineno, colno) + + if isinstance(self, SyntaxError): + syntax_error_args = (self.filename, self.lineno, self.offset, + self.text) + super(HyLanguageError, self).__init__(message, syntax_error_args) + else: + super(HyLanguageError, self).__init__(message) + + def compute_lineinfo(self, expression, filename, source, lineno, colno): + + # NOTE: We use `SyntaxError`'s field names (i.e. `text`, `offset`, + # `msg`) for compatibility and print-outs. + self.text = getattr(expression, 'source', source) + self.filename = getattr(expression, 'filename', filename) + + if self.text: + lines = self.text.splitlines() + + self.lineno = getattr(expression, 'start_line', lineno) + self.offset = getattr(expression, 'start_column', colno) + end_column = getattr(expression, 'end_column', + len(lines[self.lineno-1])) + end_line = getattr(expression, 'end_line', self.lineno) + + # Trim the source down to the essentials. + self.text = '\n'.join(lines[self.lineno-1:end_line]) + + if end_column: + if self.lineno == end_line: + self.arrow_offset = end_column + else: + self.arrow_offset = len(self.text[0]) + + self.arrow_offset -= self.offset + else: + self.arrow_offset = None + else: + # We could attempt to extract the source given a filename, but we + # don't. + self.lineno = lineno + self.offset = colno + self.arrow_offset = None + + def __str__(self): + """Provide an exception message that includes SyntaxError-like source + line information when available. + """ + global _hy_colored_errors + + # Syntax errors are special and annotate the traceback (instead of what + # we would do in the message that follows the traceback). + if isinstance(self, SyntaxError): + return super(HyLanguageError, self).__str__() + + # When there isn't extra source information, use the normal message. + if not isinstance(self, SyntaxError) and not self.text: + return super(HyLanguageError, self).__str__() + + # Re-purpose Python's builtin syntax error formatting. + output = traceback.format_exception_only( + SyntaxError, + SyntaxError(self.msg, (self.filename, self.lineno, self.offset, + self.text))) + + arrow_idx, _ = next(((i, x) for i, x in enumerate(output) + if x.strip() == '^'), + (None, None)) + if arrow_idx: + msg_idx = arrow_idx + 1 + else: + msg_idx, _ = next((i, x) for i, x in enumerate(output) + if x.startswith('SyntaxError: ')) + + # Get rid of erroneous error-type label. + output[msg_idx] = re.sub('^SyntaxError: ', '', output[msg_idx]) + + # Extend the text arrow, when given enough source info. + if arrow_idx and self.arrow_offset: + output[arrow_idx] = '{}{}^\n'.format(output[arrow_idx].rstrip('\n'), + '-' * (self.arrow_offset - 1)) + + if _hy_colored_errors: + from clint.textui import colored + output[msg_idx:] = [colored.yellow(o) for o in output[msg_idx:]] + if arrow_idx: + output[arrow_idx] = colored.green(output[arrow_idx]) + for idx, line in enumerate(output[::msg_idx]): + if line.strip().startswith( + 'File "{}", line'.format(self.filename)): + output[idx] = colored.red(line) + + # This resulting string will come after a ":" prompt, so + # put it down a line. + output.insert(0, '\n') + + # Avoid "...expected str instance, ColoredString found" + return reduce(lambda x, y: x + y, output) class HyCompileError(HyInternalError): @@ -50,88 +165,21 @@ class HyCompileError(HyInternalError): class HyTypeError(HyLanguageError, TypeError): - """TypeErrors occurring during the normal use of Hy.""" - - def __init__(self, message, filename=None, expression=None, source=None): - """ - Parameters - ---------- - message: str - The message to display for this error. - filename: str, optional - The filename for the source code generating this error. - expression: HyObject, optional - The Hy expression generating this error. - source: str, optional - The actual source code generating this error. - """ - self.message = message - self.filename = filename - self.expression = expression - self.source = source - - super(HyTypeError, self).__init__(message, filename, expression, - source) - - def __str__(self): - global _hy_colored_errors - - result = "" - - if _hy_colored_errors: - from clint.textui import colored - red, green, yellow = colored.red, colored.green, colored.yellow - else: - red = green = yellow = lambda x: x - - if all(getattr(self.expression, x, None) is not None - for x in ("start_line", "start_column", "end_column")): - - line = self.expression.start_line - start = self.expression.start_column - end = self.expression.end_column - - source = [] - if self.source is not None: - source = self.source.split("\n")[line-1:self.expression.end_line] - - if line == self.expression.end_line: - length = end - start - else: - length = len(source[0]) - start - - result += ' File "%s", line %d, column %d\n\n' % (self.filename, - line, - start) - - if len(source) == 1: - result += ' %s\n' % red(source[0]) - result += ' %s%s\n' % (' '*(start-1), - green('^' + '-'*(length-1) + '^')) - if len(source) > 1: - result += ' %s\n' % red(source[0]) - result += ' %s%s\n' % (' '*(start-1), - green('^' + '-'*length)) - if len(source) > 2: # write the middle lines - for line in source[1:-1]: - result += ' %s\n' % red("".join(line)) - result += ' %s\n' % green("-"*len(line)) - - # write the last line - result += ' %s\n' % red("".join(source[-1])) - result += ' %s\n' % green('-'*(end-1) + '^') - - else: - result += ' File "%s", unknown location\n' % self.filename - - result += yellow("%s: %s\n\n" % - (self.__class__.__name__, - self.message)) - - return result + """TypeError occurring during the normal use of Hy.""" -class HyMacroExpansionError(HyTypeError): +class HyNameError(HyLanguageError, NameError): + """NameError occurring during the normal use of Hy.""" + + +class HyRequireError(HyLanguageError): + """Errors arising during the use of `require` + + This, and any errors inheriting from this, are user-facing. + """ + + +class HyMacroExpansionError(HyLanguageError): """Errors caused by invalid use of Hy macros. This, and any errors inheriting from this, are user-facing. @@ -158,97 +206,39 @@ class HyIOError(HyInternalError, IOError): class HySyntaxError(HyLanguageError, SyntaxError): """Error during the Lexing of a Hython expression.""" - def __init__(self, message, filename=None, lineno=-1, colno=-1, - source=None): - """ - Parameters - ---------- - message: str - The exception's message. - filename: str, optional - The filename for the source code generating this error. - lineno: int, optional - The line number of the error. - colno: int, optional - The column number of the error. - source: str, optional - The actual source code generating this error. - """ - self.message = message - self.filename = filename - self.lineno = lineno - self.colno = colno - self.source = source - super(HySyntaxError, self).__init__(message, - # The builtin `SyntaxError` needs a - # tuple. - (filename, lineno, colno, source)) - @staticmethod - def from_expression(message, expression, filename=None, source=None): - if not source: - # Maybe the expression object has its own source. - source = getattr(expression, 'source', None) +def _module_filter_name(module_name): + try: + compiler_loader = pkgutil.get_loader(module_name) + if not compiler_loader: + return None + filename = compiler_loader.get_filename(module_name) if not filename: - filename = getattr(expression, 'filename', None) + return None - if source: - lineno = expression.start_line - colno = expression.start_column - end_line = getattr(expression, 'end_line', len(source)) - lines = source.splitlines() - source = '\n'.join(lines[lineno-1:end_line]) + if compiler_loader.is_package(module_name): + # Use the package directory (e.g. instead of `.../__init__.py`) so + # that we can filter all modules in a package. + return os.path.dirname(filename) else: - # We could attempt to extract the source given a filename, but we - # don't. - lineno = colno = -1 - - return HySyntaxError(message, filename, lineno, colno, source) - - def __str__(self): - global _hy_colored_errors - - output = traceback.format_exception_only(SyntaxError, - SyntaxError(*self.args)) - - if _hy_colored_errors: - from hy.errors import colored - output[-1] = colored.yellow(output[-1]) - if len(self.source) > 0: - output[-2] = colored.green(output[-2]) - for line in output[::-2]: - if line.strip().startswith( - 'File "{}", line'.format(self.filename)): - break - output[-3] = colored.red(output[-3]) - - # Avoid "...expected str instance, ColoredString found" - return reduce(lambda x, y: x + y, output) + # Normalize filename endings, because tracebacks will use `pyc` when + # the loader says `py`. + return filename.replace('.pyc', '.py') + except Exception: + return None -def _get_module_info(module): - compiler_loader = pkgutil.get_loader(module) - is_pkg = compiler_loader.is_package(module) - filename = compiler_loader.get_filename() - if is_pkg: - # Use package directory - return os.path.dirname(filename) - else: - # Normalize filename endings, because tracebacks will use `pyc` when - # the loader says `py`. - return filename.replace('.pyc', '.py') +_tb_hidden_modules = {m for m in map(_module_filter_name, + ['hy.compiler', 'hy.lex', + 'hy.cmdline', 'hy.lex.parser', + 'hy.importer', 'hy._compat', + 'hy.macros', 'hy.models', + 'rply']) + if m is not None} -_tb_hidden_modules = {_get_module_info(m) - for m in ['hy.compiler', 'hy.lex', - 'hy.cmdline', 'hy.lex.parser', - 'hy.importer', 'hy._compat', - 'hy.macros', 'hy.models', - 'rply']} - - -def hy_exc_handler(exc_type, exc_value, exc_traceback): +def hy_exc_filter(exc_type, exc_value, exc_traceback): """Produce exceptions print-outs with all frames originating from the modules in `_tb_hidden_modules` filtered out. @@ -258,20 +248,33 @@ def hy_exc_handler(exc_type, exc_value, exc_traceback): This does not remove the frames from the actual tracebacks, so debugging will show everything. """ + # frame = (filename, line number, function name*, text) + new_tb = [] + for frame in traceback.extract_tb(exc_traceback): + if not (frame[0].replace('.pyc', '.py') in _tb_hidden_modules or + os.path.dirname(frame[0]) in _tb_hidden_modules): + new_tb += [frame] + + lines = traceback.format_list(new_tb) + + if lines: + lines.insert(0, "Traceback (most recent call last):\n") + + lines.extend(traceback.format_exception_only(exc_type, exc_value)) + output = ''.join(lines) + + return output + + +def hy_exc_handler(exc_type, exc_value, exc_traceback): + """A `sys.excepthook` handler that uses `hy_exc_filter` to + remove internal Hy frames from a traceback print-out. + """ + if os.environ.get('HY_DEBUG', False): + return sys.__excepthook__(exc_type, exc_value, exc_traceback) + try: - # frame = (filename, line number, function name*, text) - new_tb = [frame for frame in traceback.extract_tb(exc_traceback) - if not (frame[0].replace('.pyc', '.py') in _tb_hidden_modules or - os.path.dirname(frame[0]) in _tb_hidden_modules)] - - lines = traceback.format_list(new_tb) - - if lines: - lines.insert(0, "Traceback (most recent call last):\n") - - lines.extend(traceback.format_exception_only(exc_type, exc_value)) - output = ''.join(lines) - + output = hy_exc_filter(exc_type, exc_value, exc_traceback) sys.stderr.write(output) sys.stderr.flush() except Exception: diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py index f1465cd..eb3ac41 100644 --- a/hy/lex/__init__.py +++ b/hy/lex/__init__.py @@ -8,10 +8,9 @@ import re import sys import unicodedata -from hy._compat import str_type, isidentifier, UCS4, reraise +from hy._compat import str_type, isidentifier, UCS4 from hy.lex.exceptions import PrematureEndOfInput, LexException # NOQA from hy.models import HyExpression, HySymbol -from hy.errors import HySyntaxError try: from io import StringIO @@ -35,15 +34,12 @@ def hy_parse(source, filename=''): out : HyExpression """ _source = re.sub(r'\A#!.*', '', source) - try: - res = HyExpression([HySymbol("do")] + - tokenize(_source + "\n", - filename=filename)) - res.source = source - res.filename = filename - return res - except HySyntaxError as e: - reraise(type(e), e, None) + res = HyExpression([HySymbol("do")] + + tokenize(_source + "\n", + filename=filename)) + res.source = source + res.filename = filename + return res class ParserState(object): @@ -70,8 +66,12 @@ def tokenize(source, filename=None): state=ParserState(source, filename)) except LexingError as e: pos = e.getsourcepos() - raise LexException("Could not identify the next token.", filename, - pos.lineno, pos.colno, source) + raise LexException("Could not identify the next token.", + None, filename, source, + max(pos.lineno, 1), + max(pos.colno, 1)) + except LexException as e: + raise e mangle_delim = 'X' diff --git a/hy/lex/exceptions.py b/hy/lex/exceptions.py index fb9aa7a..449119a 100644 --- a/hy/lex/exceptions.py +++ b/hy/lex/exceptions.py @@ -8,26 +8,31 @@ class LexException(HySyntaxError): @classmethod def from_lexer(cls, message, state, token): + lineno = None + colno = None + source = state.source source_pos = token.getsourcepos() - if token.source_pos: + + if source_pos: lineno = source_pos.lineno colno = source_pos.colno + elif source: + # Use the end of the last line of source for `PrematureEndOfInput`. + # We get rid of empty lines and spaces so that the error matches + # with the last piece of visible code. + lines = source.rstrip().splitlines() + lineno = lineno or len(lines) + colno = colno or len(lines[lineno - 1]) else: - lineno = -1 - colno = -1 + lineno = lineno or 1 + colno = colno or 1 - if state.source: - lines = state.source.splitlines() - if lines[-1] == '': - del lines[-1] - - if lineno < 1: - lineno = len(lines) - if colno < 1: - colno = len(lines[-1]) - - source = lines[lineno - 1] - return cls(message, state.filename, lineno, colno, source) + return cls(message, + None, + state.filename, + source, + lineno, + colno) class PrematureEndOfInput(LexException): diff --git a/hy/lex/parser.py b/hy/lex/parser.py index f5cd5e5..c4df2a5 100755 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -6,7 +6,6 @@ from __future__ import unicode_literals from functools import wraps -import re, unicodedata from rply import ParserGenerator @@ -278,15 +277,6 @@ def symbol_like(obj): def error_handler(state, token): tokentype = token.gettokentype() if tokentype == '$end': - source_pos = token.source_pos or token.getsourcepos() - source = state.source - if source_pos: - lineno = source_pos.lineno - colno = source_pos.colno - else: - lineno = -1 - colno = -1 - raise PrematureEndOfInput.from_lexer("Premature end of input", state, token) else: diff --git a/hy/macros.py b/hy/macros.py index 274702f..d668077 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -5,13 +5,15 @@ import sys import importlib import inspect import pkgutil +import traceback from contextlib import contextmanager -from hy._compat import PY3, string_types, reraise +from hy._compat import PY3, string_types, reraise, rename_function from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value from hy.lex import mangle -from hy.errors import HyTypeError, HyMacroExpansionError +from hy.errors import (HyLanguageError, HyMacroExpansionError, HyTypeError, + HyRequireError) try: # Check if we have the newer inspect.signature available. @@ -50,7 +52,7 @@ def macro(name): """ name = mangle(name) def _(fn): - fn.__name__ = '({})'.format(name) + fn = rename_function(fn, name) try: fn._hy_macro_pass_compiler = has_kwargs(fn) except Exception: @@ -75,7 +77,7 @@ def tag(name): if not PY3: _name = _name.encode('UTF-8') - fn.__name__ = _name + fn = rename_function(fn, _name) module = inspect.getmodule(fn) @@ -150,7 +152,6 @@ def require(source_module, target_module, assignments, prefix=""): out: boolean Whether or not macros and tags were actually transferred. """ - if target_module is None: parent_frame = inspect.stack()[1][0] target_namespace = parent_frame.f_globals @@ -161,7 +162,7 @@ def require(source_module, target_module, assignments, prefix=""): elif inspect.ismodule(target_module): target_namespace = target_module.__dict__ else: - raise TypeError('`target_module` is not a recognized type: {}'.format( + raise HyTypeError('`target_module` is not a recognized type: {}'.format( type(target_module))) # Let's do a quick check to make sure the source module isn't actually @@ -173,14 +174,17 @@ def require(source_module, target_module, assignments, prefix=""): return False if not inspect.ismodule(source_module): - source_module = importlib.import_module(source_module) + try: + source_module = importlib.import_module(source_module) + except ImportError as e: + reraise(HyRequireError, HyRequireError(e.args[0]), None) source_macros = source_module.__dict__.setdefault('__macros__', {}) source_tags = source_module.__dict__.setdefault('__tags__', {}) if len(source_module.__macros__) + len(source_module.__tags__) == 0: if assignments != "ALL": - raise ImportError('The module {} has no macros or tags'.format( + raise HyRequireError('The module {} has no macros or tags'.format( source_module)) else: return False @@ -205,7 +209,7 @@ def require(source_module, target_module, assignments, prefix=""): elif _name in source_module.__tags__: target_tags[alias] = source_tags[_name] else: - raise ImportError('Could not require name {} from {}'.format( + raise HyRequireError('Could not require name {} from {}'.format( _name, source_module)) return True @@ -239,50 +243,33 @@ def load_macros(module): if k not in module_tags}) -def make_empty_fn_copy(fn): - try: - # This might fail if fn has parameters with funny names, like o!n. In - # such a case, we return a generic function that ensures the program - # can continue running. Unfortunately, the error message that might get - # raised later on while expanding a macro might not make sense at all. - - formatted_args = format_args(fn) - fn_str = 'lambda {}: None'.format( - formatted_args.lstrip('(').rstrip(')')) - empty_fn = eval(fn_str) - - except Exception: - - def empty_fn(*args, **kwargs): - None - - return empty_fn - - @contextmanager def macro_exceptions(module, macro_tree, compiler=None): try: yield + except HyLanguageError as e: + # These are user-level Hy errors occurring in the macro. + # We want to pass them up to the user. + reraise(type(e), e, sys.exc_info()[2]) except Exception as e: - try: - filename = inspect.getsourcefile(module) - source = inspect.getsource(module) - except TypeError: - if compiler: - filename = compiler.filename - source = compiler.source - if not isinstance(e, HyTypeError): - exc_type = HyMacroExpansionError - msg = "expanding `{}': ".format(macro_tree[0]) - msg += str(e).replace("()", "", 1).strip() + if compiler: + filename = compiler.filename + source = compiler.source else: - exc_type = HyTypeError - msg = e.message + filename = None + source = None - reraise(exc_type, - exc_type(msg, filename, macro_tree, source), - sys.exc_info()[2].tb_next) + exc_msg = ' '.join(traceback.format_exception_only( + sys.exc_info()[0], sys.exc_info()[1])) + + msg = "expanding macro {}\n ".format(str(macro_tree[0])) + msg += exc_msg + + reraise(HyMacroExpansionError, + HyMacroExpansionError( + msg, macro_tree, filename, source), + sys.exc_info()[2]) def macroexpand(tree, module, compiler=None, once=False): @@ -353,8 +340,6 @@ def macroexpand(tree, module, compiler=None, once=False): opts['compiler'] = compiler with macro_exceptions(module, tree, compiler): - m_copy = make_empty_fn_copy(m) - m_copy(module.__name__, *tree[1:], **opts) obj = m(module.__name__, *tree[1:], **opts) if isinstance(obj, HyExpression): diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 0c7bcd5..9311eef 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -6,9 +6,8 @@ from __future__ import unicode_literals from hy import HyString -from hy.models import HyObject from hy.compiler import hy_compile, hy_eval -from hy.errors import HyCompileError, HyTypeError +from hy.errors import HyCompileError, HyLanguageError, HyError from hy.lex import hy_parse from hy.lex.exceptions import LexException, PrematureEndOfInput from hy._compat import PY3 @@ -27,7 +26,7 @@ def _ast_spotcheck(arg, root, secondary): def can_compile(expr): - return hy_compile(hy_parse(expr), "__main__") + return hy_compile(hy_parse(expr), __name__) def can_eval(expr): @@ -35,21 +34,16 @@ def can_eval(expr): def cant_compile(expr): - try: - hy_compile(hy_parse(expr), "__main__") - assert False - except HyTypeError as e: + with pytest.raises(HyError) as excinfo: + hy_compile(hy_parse(expr), __name__) + + if issubclass(excinfo.type, HyLanguageError): + assert excinfo.value.msg + return excinfo.value + elif issubclass(excinfo.type, HyCompileError): # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. - assert isinstance(e.expression, HyObject) - assert e.message - return e - except HyCompileError as e: - # Anything that can't be compiled should raise a user friendly - # error, otherwise it's a compiler bug. - assert isinstance(e.exception, HyTypeError) - assert e.traceback - return e + return excinfo.value def s(x): @@ -60,11 +54,9 @@ def test_ast_bad_type(): "Make sure AST breakage can happen" class C: pass - try: - hy_compile(C(), "__main__") - assert True is False - except TypeError: - pass + + with pytest.raises(TypeError): + hy_compile(C(), __name__, filename='', source='') def test_empty_expr(): @@ -473,7 +465,7 @@ def test_lambda_list_keywords_kwonly(): assert code.body[0].args.kw_defaults[1].n == 2 else: exception = cant_compile(kwonly_demo) - assert isinstance(exception, HyTypeError) + assert isinstance(exception, HyLanguageError) message = exception.args[0] assert message == "&kwonly parameters require Python 3" @@ -489,9 +481,9 @@ def test_lambda_list_keywords_mixed(): def test_missing_keyword_argument_value(): """Ensure the compiler chokes on missing keyword argument values.""" - with pytest.raises(HyTypeError) as excinfo: + with pytest.raises(HyLanguageError) as excinfo: can_compile("((fn [x] x) :x)") - assert excinfo.value.message == "Keyword argument :x needs a value." + assert excinfo.value.msg == "Keyword argument :x needs a value." def test_ast_unicode_strings(): @@ -500,7 +492,7 @@ def test_ast_unicode_strings(): def _compile_string(s): hy_s = HyString(s) - code = hy_compile([hy_s], "__main__") + code = hy_compile([hy_s], __name__, filename='', source=s) # We put hy_s in a list so it isn't interpreted as a docstring. # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))]) @@ -541,7 +533,7 @@ Only one leading newline should be removed. def test_compile_error(): """Ensure we get compile error in tricky cases""" - with pytest.raises(HyTypeError) as excinfo: + with pytest.raises(HyLanguageError) as excinfo: can_compile("(fn [] (in [1 2 3]))") @@ -549,11 +541,11 @@ def test_for_compile_error(): """Ensure we get compile error in tricky 'for' cases""" with pytest.raises(PrematureEndOfInput) as excinfo: can_compile("(fn [] (for)") - assert excinfo.value.message == "Premature end of input" + assert excinfo.value.msg == "Premature end of input" with pytest.raises(LexException) as excinfo: can_compile("(fn [] (for)))") - assert excinfo.value.message == "Ran into a RPAREN where it wasn't expected." + assert excinfo.value.msg == "Ran into a RPAREN where it wasn't expected." cant_compile("(fn [] (for [x] x))") @@ -605,13 +597,13 @@ def test_setv_builtins(): def test_top_level_unquote(): - with pytest.raises(HyTypeError) as excinfo: + with pytest.raises(HyLanguageError) as excinfo: can_compile("(unquote)") - assert excinfo.value.message == "The special form 'unquote' is not allowed here" + assert excinfo.value.msg == "The special form 'unquote' is not allowed here" - with pytest.raises(HyTypeError) as excinfo: + with pytest.raises(HyLanguageError) as excinfo: can_compile("(unquote-splice)") - assert excinfo.value.message == "The special form 'unquote-splice' is not allowed here" + assert excinfo.value.msg == "The special form 'unquote-splice' is not allowed here" def test_lots_of_comment_lines(): diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py index 309ca49..134bd5b 100644 --- a/tests/macros/test_macro_processor.py +++ b/tests/macros/test_macro_processor.py @@ -50,8 +50,7 @@ def test_preprocessor_exceptions(): """ Test that macro expansion raises appropriate exceptions""" with pytest.raises(HyMacroExpansionError) as excinfo: macroexpand(tokenize('(defn)')[0], __name__, HyASTCompiler(__name__)) - assert "_hy_anon_fn_" not in excinfo.value.message - assert "TypeError" not in excinfo.value.message + assert "_hy_anon_" not in excinfo.value.msg def test_macroexpand_nan(): diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy index a89eece..ab8eaf4 100644 --- a/tests/native_tests/core.hy +++ b/tests/native_tests/core.hy @@ -687,5 +687,5 @@ result['y in globals'] = 'y' in globals()") (doc doc) (setv out_err (.readouterr capsys)) (assert (.startswith (.strip (first out_err)) - "Help on function (doc) in module hy.core.macros:")) + "Help on function doc in module hy.core.macros:")) (assert (empty? (second out_err)))) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index c237a5a..65c629a 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -7,7 +7,7 @@ [sys :as systest] re [operator [or_]] - [hy.errors [HyTypeError]] + [hy.errors [HyLanguageError]] pytest) (import sys) @@ -68,16 +68,16 @@ "NATIVE: test that setv doesn't work on names Python can't assign to and that we can't mangle" (try (eval '(setv None 1)) - (except [e [TypeError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) (try (eval '(defn None [] (print "hello"))) - (except [e [TypeError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) (when PY3 (try (eval '(setv False 1)) - (except [e [TypeError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) (try (eval '(setv True 0)) - (except [e [TypeError]] (assert (in "Can't assign to" (str e))))) + (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))) (try (eval '(defn True [] (print "hello"))) - (except [e [TypeError]] (assert (in "Can't assign to" (str e))))))) + (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))))) (defn test-setv-pairs [] @@ -87,7 +87,7 @@ (assert (= b 2)) (setv y 0 x 1 y x) (assert (= y 1)) - (with [(pytest.raises HyTypeError)] + (with [(pytest.raises HyLanguageError)] (eval '(setv a 1 b)))) @@ -144,29 +144,29 @@ (do (eval '(setv (do 1 2) 1)) (assert False)) - (except [e HyTypeError] - (assert (= e.message "Can't assign or delete a non-expression")))) + (except [e HyLanguageError] + (assert (= e.msg "Can't assign or delete a non-expression")))) (try (do (eval '(setv 1 1)) (assert False)) - (except [e HyTypeError] - (assert (= e.message "Can't assign or delete a HyInteger")))) + (except [e HyLanguageError] + (assert (= e.msg "Can't assign or delete a HyInteger")))) (try (do (eval '(setv {1 2} 1)) (assert False)) - (except [e HyTypeError] - (assert (= e.message "Can't assign or delete a HyDict")))) + (except [e HyLanguageError] + (assert (= e.msg "Can't assign or delete a HyDict")))) (try (do (eval '(del 1 1)) (assert False)) - (except [e HyTypeError] - (assert (= e.message "Can't assign or delete a HyInteger"))))) + (except [e HyLanguageError] + (assert (= e.msg "Can't assign or delete a HyInteger"))))) (defn test-no-str-as-sym [] diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index 1700b5d..6759f42 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -3,7 +3,7 @@ ;; license. See the LICENSE. (import pytest - [hy.errors [HyTypeError]]) + [hy.errors [HyTypeError HyMacroExpansionError]]) (defmacro rev [&rest body] "Execute the `body` statements in reverse" @@ -66,13 +66,13 @@ (try (eval '(defmacro f [&kwonly a b])) (except [e HyTypeError] - (assert (= e.message "macros cannot use &kwonly"))) + (assert (= e.msg "macros cannot use &kwonly"))) (else (assert False))) (try (eval '(defmacro f [&kwargs kw])) (except [e HyTypeError] - (assert (= e.message "macros cannot use &kwargs"))) + (assert (= e.msg "macros cannot use &kwargs"))) (else (assert False)))) (defn test-fn-calling-macro [] @@ -483,3 +483,28 @@ in expansions." (test-macro) (assert (= blah 1))) + + +(defn test-macro-errors [] + (import traceback + [hy.importer [hy-parse]]) + + (setv test-expr (hy-parse "(defmacro blah [x] `(print ~@z)) (blah y)")) + + (with [excinfo (pytest.raises HyMacroExpansionError)] + (eval test-expr)) + + (setv output (traceback.format_exception_only + excinfo.type excinfo.value)) + (setv output (cut (.splitlines (.strip (first output))) 1)) + + (setv expected [" File \"\", line 1" + " (defmacro blah [x] `(print ~@z)) (blah y)" + " ^------^" + "expanding macro blah" + " NameError: global name 'z' is not defined"]) + + (assert (= (cut expected 0 -1) (cut output 0 -1))) + (assert (or (= (get expected -1) (get output -1)) + ;; Handle PyPy's peculiarities + (= (.replace (get expected -1) "global " "") (get output -1))))) diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy index 716eb77..e08edbb 100644 --- a/tests/native_tests/operators.hy +++ b/tests/native_tests/operators.hy @@ -28,7 +28,7 @@ (defmacro forbid [expr] `(assert (try (eval '~expr) - (except [TypeError] True) + (except [[TypeError SyntaxError]] True) (else (raise AssertionError))))) diff --git a/tests/test_bin.py b/tests/test_bin.py index 8aef923..aad45e5 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -11,6 +11,7 @@ import shlex import subprocess from hy.importer import cache_from_source +from hy._compat import PY3 import pytest @@ -123,7 +124,16 @@ def test_bin_hy_stdin_as_arrow(): def test_bin_hy_stdin_error_underline_alignment(): _, err = run_cmd("hy", "(defmacro mabcdefghi [x] x)\n(mabcdefghi)") - assert "\n (mabcdefghi)\n ^----------^" in err + + msg_idx = err.rindex(" (mabcdefghi)") + assert msg_idx + err_parts = err[msg_idx:].splitlines() + assert err_parts[1].startswith(" ^----------^") + assert err_parts[2].startswith("expanding macro mabcdefghi") + assert (err_parts[3].startswith(" TypeError: mabcdefghi") or + # PyPy can use a function's `__name__` instead of + # `__code__.co_name`. + err_parts[3].startswith(" TypeError: (mabcdefghi)")) def test_bin_hy_stdin_except_do(): @@ -153,6 +163,62 @@ def test_bin_hy_stdin_unlocatable_hytypeerror(): assert "AZ" in err +def test_bin_hy_error_parts_length(): + """Confirm that exception messages print arrows surrounding the affected + expression.""" + prg_str = """ + (import hy.errors + [hy.importer [hy-parse]]) + + (setv test-expr (hy-parse "(+ 1\n\n'a 2 3\n\n 1)")) + (setv test-expr.start-line {}) + (setv test-expr.start-column {}) + (setv test-expr.end-column {}) + + (raise (hy.errors.HyLanguageError + "this\nis\na\nmessage" + test-expr + None + None)) + """ + + # Up-arrows right next to each other. + _, err = run_cmd("hy", prg_str.format(3, 1, 2)) + + msg_idx = err.rindex("HyLanguageError:") + assert msg_idx + err_parts = err[msg_idx:].splitlines()[1:] + + expected = [' File "", line 3', + ' \'a 2 3', + ' ^^', + 'this', + 'is', + 'a', + 'message'] + + for obs, exp in zip(err_parts, expected): + assert obs.startswith(exp) + + # Make sure only one up-arrow is printed + _, err = run_cmd("hy", prg_str.format(3, 1, 1)) + + msg_idx = err.rindex("HyLanguageError:") + assert msg_idx + err_parts = err[msg_idx:].splitlines()[1:] + assert err_parts[2] == ' ^' + + # Make sure lines are printed in between arrows separated by more than one + # character. + _, err = run_cmd("hy", prg_str.format(3, 1, 6)) + print(err) + + msg_idx = err.rindex("HyLanguageError:") + assert msg_idx + err_parts = err[msg_idx:].splitlines()[1:] + assert err_parts[2] == ' ^----^' + + def test_bin_hy_stdin_bad_repr(): # https://github.com/hylang/hy/issues/1389 output, err = run_cmd("hy", """ @@ -423,3 +489,86 @@ def test_bin_hy_macro_require(): assert os.path.exists(cache_from_source(test_file)) output, _ = run_cmd("hy {}".format(test_file)) assert "abc" == output.strip() + + +def test_bin_hy_tracebacks(): + """Make sure the printed tracebacks are correct.""" + + # We want the filtered tracebacks. + os.environ['HY_DEBUG'] = '' + + def req_err(x): + assert x == '{}HyRequireError: No module named {}'.format( + 'hy.errors.' if PY3 else '', + (repr if PY3 else str)('not_a_real_module')) + + # Modeled after + # > python -c 'import not_a_real_module' + # Traceback (most recent call last): + # File "", line 1, in + # ImportError: No module named not_a_real_module + _, error = run_cmd('hy', '(require not-a-real-module)') + error_lines = error.splitlines() + if error_lines[-1] == '': + del error_lines[-1] + assert len(error_lines) <= 10 + # Rough check for the internal traceback filtering + req_err(error_lines[4 if PY3 else -1]) + + _, error = run_cmd('hy -c "(require not-a-real-module)"', expect=1) + error_lines = error.splitlines() + assert len(error_lines) <= 4 + req_err(error_lines[-1]) + + output, error = run_cmd('hy -i "(require not-a-real-module)"') + assert output.startswith('=> ') + print(error.splitlines()) + req_err(error.splitlines()[3 if PY3 else -3]) + + # Modeled after + # > python -c 'print("hi' + # File "", line 1 + # print("hi + # ^ + # SyntaxError: EOL while scanning string literal + _, error = run_cmd(r'hy -c "(print \""', expect=1) + peoi = ( + ' File "", line 1\n' + ' (print "\n' + ' ^\n' + + '{}PrematureEndOfInput: Partial string literal\n'.format( + 'hy.lex.exceptions.' if PY3 else '')) + assert error == peoi + + # Modeled after + # > python -i -c "print('" + # File "", line 1 + # print(' + # ^ + # SyntaxError: EOL while scanning string literal + # >>> + output, error = run_cmd(r'hy -i "(print \""') + assert output.startswith('=> ') + assert error.startswith(peoi) + + # Modeled after + # > python -c 'print(a)' + # Traceback (most recent call last): + # File "", line 1, in + # NameError: name 'a' is not defined + output, error = run_cmd('hy -c "(print a)"', expect=1) + error_lines = error.splitlines() + assert error_lines[3] == ' File "", line 1, in ' + # PyPy will add "global" to this error message, so we work around that. + assert error_lines[-1].strip().replace(' global', '') == ( + "NameError: name 'a' is not defined") + + # Modeled after + # > python -c 'compile()' + # Traceback (most recent call last): + # File "", line 1, in + # TypeError: Required argument 'source' (pos 1) not found + output, error = run_cmd('hy -c "(compile)"', expect=1) + error_lines = error.splitlines() + assert error_lines[-2] == ' File "", line 1, in ' + assert error_lines[-1].startswith('TypeError') diff --git a/tests/test_lex.py b/tests/test_lex.py index b0a03dc..f709719 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -61,7 +61,7 @@ def test_lex_single_quote_err(): with lexe() as execinfo: tokenize("' ") check_ex(execinfo, [ - ' File "", line -1\n', + ' File "", line 1\n', " '\n", ' ^\n', 'LexException: Could not identify the next token.\n']) @@ -472,7 +472,7 @@ def test_lex_exception_filtering(capsys): # First, test for PrematureEndOfInput with peoi() as execinfo: - tokenize(" \n (foo") + tokenize(" \n (foo\n \n") check_trace_output(capsys, execinfo, [ ' File "", line 2', ' (foo', From 4ae4baac2a0fd898cd83345231287f9795aa6f8c Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Thu, 1 Nov 2018 16:40:13 -0500 Subject: [PATCH 6/8] Cache command line source for exceptions Source entered interactively can now be displayed in traceback output. Also, the REPL object is now available in its namespace, so that, for instance, display options--like `spy`--can be turned on and off interactively. Closes hylang/hy#1397. --- hy/_compat.py | 5 +- hy/cmdline.py | 171 ++++++++++++++++++++++++++++++++++++++++++---- hy/errors.py | 3 +- tests/test_bin.py | 20 +++--- 4 files changed, 168 insertions(+), 31 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index 2711445..92aa392 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -28,10 +28,7 @@ string_types = str if PY3 else basestring # NOQA if PY3: raise_src = textwrap.dedent(''' def raise_from(value, from_value): - try: - raise value from from_value - finally: - traceback = None + raise value from from_value ''') def reraise(exc_type, value, traceback=None): diff --git a/hy/cmdline.py b/hy/cmdline.py index a9f3af3..f65379d 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -15,13 +15,20 @@ import py_compile import traceback import runpy import types +import time +import linecache +import hashlib +import codeop import astor.code_gen import hy + from hy.lex import hy_parse, mangle +from contextlib import contextmanager from hy.lex.exceptions import PrematureEndOfInput -from hy.compiler import HyASTCompiler, hy_eval, hy_compile, ast_compile +from hy.compiler import (HyASTCompiler, hy_eval, hy_compile, + hy_ast_compile_flags) from hy.errors import (HyLanguageError, HyRequireError, HyMacroExpansionError, filtered_hy_exceptions, hy_exc_handler) from hy.importer import runhy @@ -31,6 +38,11 @@ from hy.models import HyExpression, HyString, HySymbol from hy._compat import builtins, PY3, FileNotFoundError +sys.last_type = None +sys.last_value = None +sys.last_traceback = None + + class HyQuitter(object): def __init__(self, name): self.name = name @@ -51,14 +63,101 @@ class HyQuitter(object): builtins.quit = HyQuitter('quit') builtins.exit = HyQuitter('exit') +@contextmanager +def extend_linecache(add_cmdline_cache): + _linecache_checkcache = linecache.checkcache -class HyCommandCompiler(object): - def __init__(self, module, ast_callback=None, hy_compiler=None): + def _cmdline_checkcache(*args): + _linecache_checkcache(*args) + linecache.cache.update(add_cmdline_cache) + + linecache.checkcache = _cmdline_checkcache + yield + linecache.checkcache = _linecache_checkcache + + +_codeop_maybe_compile = codeop._maybe_compile + + +def _hy_maybe_compile(compiler, source, filename, symbol): + """The `codeop` version of this will compile the same source multiple + times, and, since we have macros and things like `eval-and-compile`, we + can't allow that. + """ + if not isinstance(compiler, HyCompile): + return _codeop_maybe_compile(compiler, source, filename, symbol) + + for line in source.split("\n"): + line = line.strip() + if line and line[0] != ';': + # Leave it alone (could do more with Hy syntax) + break + else: + if symbol != "eval": + # Replace it with a 'pass' statement (i.e. tell the compiler to do + # nothing) + source = "pass" + + return compiler(source, filename, symbol) + + +codeop._maybe_compile = _hy_maybe_compile + + +class HyCompile(codeop.Compile, object): + """This compiler uses `linecache` like + `IPython.core.compilerop.CachingCompiler`. + """ + + def __init__(self, module, locals, ast_callback=None, + hy_compiler=None, cmdline_cache={}): self.module = module + self.locals = locals self.ast_callback = ast_callback self.hy_compiler = hy_compiler + super(HyCompile, self).__init__() + + self.flags |= hy_ast_compile_flags + + self.cmdline_cache = cmdline_cache + + def _cache(self, source, name): + entry = (len(source), + time.time(), + [line + '\n' for line in source.splitlines()], + name) + + linecache.cache[name] = entry + self.cmdline_cache[name] = entry + + def _update_exc_info(self): + self.locals['_hy_last_type'] = sys.last_type + self.locals['_hy_last_value'] = sys.last_value + # Skip our frame. + sys.last_traceback = getattr(sys.last_traceback, 'tb_next', + sys.last_traceback) + self.locals['_hy_last_traceback'] = sys.last_traceback + def __call__(self, source, filename="", symbol="single"): + + if source == 'pass': + # We need to return a no-op to signal that no more input is needed. + return (compile(source, filename, symbol),) * 2 + + hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest() + name = '{}-{}'.format(filename.strip('<>'), hash_digest) + + try: + hy_ast = hy_parse(source, filename=name) + except Exception: + # Capture a traceback without the compiler/REPL frames. + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + self._update_exc_info() + raise + + self._cache(source, name) + try: hy_ast = hy_parse(source, filename=filename) root_ast = ast.Interactive if symbol == 'single' else ast.Module @@ -75,14 +174,39 @@ class HyCommandCompiler(object): if self.ast_callback: self.ast_callback(exec_ast, eval_ast) - exec_code = ast_compile(exec_ast, filename, symbol) - eval_code = ast_compile(eval_ast, filename, 'eval') + exec_code = super(HyCompile, self).__call__(exec_ast, name, symbol) + eval_code = super(HyCompile, self).__call__(eval_ast, name, 'eval') - return exec_code, eval_code - except PrematureEndOfInput: - # Save these so that we can reraise/display when an incomplete - # interactive command is given at the prompt. + except HyLanguageError: + # Hy will raise exceptions during compile-time that Python would + # raise during run-time (e.g. import errors for `require`). In + # order to work gracefully with the Python world, we convert such + # Hy errors to code that purposefully reraises those exceptions in + # the places where Python code expects them. sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + self._update_exc_info() + exec_code = super(HyCompile, self).__call__( + 'import hy._compat; hy._compat.reraise(' + '_hy_last_type, _hy_last_value, _hy_last_traceback)', + name, symbol) + eval_code = super(HyCompile, self).__call__('None', name, 'eval') + + return exec_code, eval_code + + +class HyCommandCompiler(codeop.CommandCompiler, object): + def __init__(self, *args, **kwargs): + self.compiler = HyCompile(*args, **kwargs) + + def __call__(self, *args, **kwargs): + try: + return super(HyCommandCompiler, self).__call__(*args, **kwargs) + except PrematureEndOfInput: + # We have to do this here, because `codeop._maybe_compile` won't + # take `None` for a return value (at least not in Python 2.7) and + # this exception type is also a `SyntaxError`, so it will be caught + # by `code.InteractiveConsole` base methods before it reaches our + # `runsource`. return None @@ -111,11 +235,16 @@ class HyREPL(code.InteractiveConsole, object): self.hy_compiler = HyASTCompiler(self.module) - self.compile = HyCommandCompiler(self.module, self.ast_callback, - self.hy_compiler) + self.cmdline_cache = {} + self.compile = HyCommandCompiler(self.module, + self.locals, + ast_callback=self.ast_callback, + hy_compiler=self.hy_compiler, + cmdline_cache=self.cmdline_cache) self.spy = spy self.last_value = None + self.print_last_value = True if output_fn is None: self.output_fn = repr @@ -133,6 +262,9 @@ class HyREPL(code.InteractiveConsole, object): self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] self.locals.update({sym: None for sym in self._repl_results_symbols}) + # Allow access to the running REPL instance + self.locals['_hy_repl'] = self + def ast_callback(self, exec_ast, eval_ast): if self.spy: try: @@ -146,11 +278,17 @@ class HyREPL(code.InteractiveConsole, object): traceback.format_exc()) self.write(msg) - def _error_wrap(self, error_fn, *args, **kwargs): + def _error_wrap(self, error_fn, exc_info_override=False, *args, **kwargs): sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() - # Sadly, this method in Python 2.7 ignores an overridden - # `sys.excepthook`. + if exc_info_override: + # Use a traceback that doesn't have the REPL frames. + sys.last_type = self.locals.get('_hy_last_type', sys.last_type) + sys.last_value = self.locals.get('_hy_last_value', sys.last_value) + sys.last_traceback = self.locals.get('_hy_last_traceback', + sys.last_traceback) + + # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`. if sys.excepthook is sys.__excepthook__: error_fn(*args, **kwargs) else: @@ -163,6 +301,7 @@ class HyREPL(code.InteractiveConsole, object): filename = self.filename self._error_wrap(super(HyREPL, self).showsyntaxerror, + exc_info_override=True, filename=filename) def showtraceback(self): @@ -289,7 +428,9 @@ def run_repl(hr=None, **kwargs): hr = HyREPL(**kwargs) namespace = hr.locals - with filtered_hy_exceptions(), completion(Completer(namespace)): + with filtered_hy_exceptions(), \ + extend_linecache(hr.cmdline_cache), \ + completion(Completer(namespace)): hr.interact("{appname} {version} using " "{py}({build}) {pyversion} on {os}".format( appname=hy.__appname__, diff --git a/hy/errors.py b/hy/errors.py index 0579e96..a0cd589 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -257,8 +257,7 @@ def hy_exc_filter(exc_type, exc_value, exc_traceback): lines = traceback.format_list(new_tb) - if lines: - lines.insert(0, "Traceback (most recent call last):\n") + lines.insert(0, "Traceback (most recent call last):\n") lines.extend(traceback.format_exception_only(exc_type, exc_value)) output = ''.join(lines) diff --git a/tests/test_bin.py b/tests/test_bin.py index aad45e5..06b3af9 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -6,7 +6,6 @@ import os import re -import sys import shlex import subprocess @@ -523,7 +522,7 @@ def test_bin_hy_tracebacks(): output, error = run_cmd('hy -i "(require not-a-real-module)"') assert output.startswith('=> ') print(error.splitlines()) - req_err(error.splitlines()[3 if PY3 else -3]) + req_err(error.splitlines()[2 if PY3 else -3]) # Modeled after # > python -c 'print("hi' @@ -532,13 +531,14 @@ def test_bin_hy_tracebacks(): # ^ # SyntaxError: EOL while scanning string literal _, error = run_cmd(r'hy -c "(print \""', expect=1) - peoi = ( - ' File "", line 1\n' - ' (print "\n' - ' ^\n' + - '{}PrematureEndOfInput: Partial string literal\n'.format( - 'hy.lex.exceptions.' if PY3 else '')) - assert error == peoi + peoi_re = ( + r'Traceback \(most recent call last\):\n' + r' File "(?:|string-[0-9a-f]+)", line 1\n' + r' \(print "\n' + r' \^\n' + + r'{}PrematureEndOfInput: Partial string literal\n'.format( + r'hy\.lex\.exceptions\.' if PY3 else '')) + assert re.search(peoi_re, error) # Modeled after # > python -i -c "print('" @@ -549,7 +549,7 @@ def test_bin_hy_tracebacks(): # >>> output, error = run_cmd(r'hy -i "(print \""') assert output.startswith('=> ') - assert error.startswith(peoi) + assert re.match(peoi_re, error) # Modeled after # > python -c 'print(a)' From 9e62903d8aa8a22f18bd58790762ec63b396dcde Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Tue, 13 Nov 2018 12:11:51 -0600 Subject: [PATCH 7/8] Add special exception and handling for wrapper errors --- hy/errors.py | 9 +++++++++ hy/macros.py | 6 +++--- hy/models.py | 3 ++- tests/native_tests/native_macros.hy | 31 +++++++++++++++++++---------- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/hy/errors.py b/hy/errors.py index a0cd589..0b7619e 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -207,6 +207,15 @@ class HySyntaxError(HyLanguageError, SyntaxError): """Error during the Lexing of a Hython expression.""" +class HyWrapperError(HyError, TypeError): + """Errors caused by language model object wrapping. + + These can be caused by improper user-level use of a macro, so they're + not really "internal". If they arise due to anything else, they're an + internal/compiler problem, though. + """ + + def _module_filter_name(module_name): try: compiler_loader = pkgutil.get_loader(module_name) diff --git a/hy/macros.py b/hy/macros.py index d668077..e2cec31 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -342,10 +342,10 @@ def macroexpand(tree, module, compiler=None, once=False): with macro_exceptions(module, tree, compiler): obj = m(module.__name__, *tree[1:], **opts) - if isinstance(obj, HyExpression): - obj.module = inspect.getmodule(m) + if isinstance(obj, HyExpression): + obj.module = inspect.getmodule(m) - tree = replace_hy_obj(obj, tree) + tree = replace_hy_obj(obj, tree) if once: break diff --git a/hy/models.py b/hy/models.py index 134e322..478c691 100644 --- a/hy/models.py +++ b/hy/models.py @@ -7,6 +7,7 @@ from contextlib import contextmanager from math import isnan, isinf from hy import _initialize_env_var from hy._compat import PY3, str_type, bytes_type, long_type, string_types +from hy.errors import HyWrapperError from fractions import Fraction from clint.textui import colored @@ -64,7 +65,7 @@ def wrap_value(x): new = _wrappers.get(type(x), lambda y: y)(x) if not isinstance(new, HyObject): - raise TypeError("Don't know how to wrap {!r}: {!r}".format(type(x), x)) + raise HyWrapperError("Don't know how to wrap {!r}: {!r}".format(type(x), x)) if isinstance(x, HyObject): new = new.replace(x, recursive=False) if not hasattr(new, "start_column"): diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index 6759f42..d5c48c7 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -162,8 +162,8 @@ ") ;; expand the macro twice, should use a different ;; gensym each time - (setv _ast1 (hy-compile (hy-parse macro1) "foo")) - (setv _ast2 (hy-compile (hy-parse macro1) "foo")) + (setv _ast1 (hy-compile (hy-parse macro1) __name__)) + (setv _ast2 (hy-compile (hy-parse macro1) __name__)) (setv s1 (to_source _ast1)) (setv s2 (to_source _ast2)) ;; and make sure there is something new that starts with _;G| @@ -189,8 +189,8 @@ ") ;; expand the macro twice, should use a different ;; gensym each time - (setv _ast1 (hy-compile (hy-parse macro1) "foo")) - (setv _ast2 (hy-compile (hy-parse macro1) "foo")) + (setv _ast1 (hy-compile (hy-parse macro1) __name__)) + (setv _ast2 (hy-compile (hy-parse macro1) __name__)) (setv s1 (to_source _ast1)) (setv s2 (to_source _ast2)) (assert (in (mangle "_;a|") s1)) @@ -213,8 +213,8 @@ ") ;; expand the macro twice, should use a different ;; gensym each time - (setv _ast1 (hy-compile (hy-parse macro1) "foo")) - (setv _ast2 (hy-compile (hy-parse macro1) "foo")) + (setv _ast1 (hy-compile (hy-parse macro1) __name__)) + (setv _ast2 (hy-compile (hy-parse macro1) __name__)) (setv s1 (to_source _ast1)) (setv s2 (to_source _ast2)) (assert (in (mangle "_;res|") s1)) @@ -224,7 +224,7 @@ ;; defmacro/g! didn't like numbers initially because they ;; don't have a startswith method and blew up during expansion (setv macro2 "(defmacro/g! two-point-zero [] `(+ (float 1) 1.0))") - (assert (hy-compile (hy-parse macro2) "foo"))) + (assert (hy-compile (hy-parse macro2) __name__))) (defn test-defmacro! [] ;; defmacro! must do everything defmacro/g! can @@ -243,8 +243,8 @@ ") ;; expand the macro twice, should use a different ;; gensym each time - (setv _ast1 (hy-compile (hy-parse macro1) "foo")) - (setv _ast2 (hy-compile (hy-parse macro1) "foo")) + (setv _ast1 (hy-compile (hy-parse macro1) __name__)) + (setv _ast2 (hy-compile (hy-parse macro1) __name__)) (setv s1 (to_source _ast1)) (setv s2 (to_source _ast2)) (assert (in (mangle "_;res|") s1)) @@ -254,7 +254,7 @@ ;; defmacro/g! didn't like numbers initially because they ;; don't have a startswith method and blew up during expansion (setv macro2 "(defmacro! two-point-zero [] `(+ (float 1) 1.0))") - (assert (hy-compile (hy-parse macro2) "foo")) + (assert (hy-compile (hy-parse macro2) __name__)) (defmacro! foo! [o!foo] `(do ~g!foo ~g!foo)) ;; test that o! becomes g! @@ -507,4 +507,13 @@ in expansions." (assert (= (cut expected 0 -1) (cut output 0 -1))) (assert (or (= (get expected -1) (get output -1)) ;; Handle PyPy's peculiarities - (= (.replace (get expected -1) "global " "") (get output -1))))) + (= (.replace (get expected -1) "global " "") (get output -1)))) + + + ;; This should throw a `HyWrapperError` that gets turned into a + ;; `HyMacroExpansionError`. + (with [excinfo (pytest.raises HyMacroExpansionError)] + (eval '(do (defmacro wrap-error-test [] + (fn [])) + (wrap-error-test)))) + (assert (in "HyWrapperError" (str excinfo.value)))) From 96d1b9c3face94d62711a6215a72722530ac588d Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 7 Feb 2019 13:50:30 -0500 Subject: [PATCH 8/8] Update README --- NEWS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.rst b/NEWS.rst index 6f3d53c..20d45ef 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -21,6 +21,7 @@ New Features Bug Fixes ------------------------------ +* Cleaned up syntax and compiler errors * Fixed issue with empty arguments in `defmain`. * `require` now compiles to Python AST. * Fixed circular `require`s.