From 4ae4baac2a0fd898cd83345231287f9795aa6f8c Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Thu, 1 Nov 2018 16:40:13 -0500 Subject: [PATCH] Cache command line source for exceptions Source entered interactively can now be displayed in traceback output. Also, the REPL object is now available in its namespace, so that, for instance, display options--like `spy`--can be turned on and off interactively. Closes hylang/hy#1397. --- hy/_compat.py | 5 +- hy/cmdline.py | 171 ++++++++++++++++++++++++++++++++++++++++++---- hy/errors.py | 3 +- tests/test_bin.py | 20 +++--- 4 files changed, 168 insertions(+), 31 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index 2711445..92aa392 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -28,10 +28,7 @@ string_types = str if PY3 else basestring # NOQA if PY3: raise_src = textwrap.dedent(''' def raise_from(value, from_value): - try: - raise value from from_value - finally: - traceback = None + raise value from from_value ''') def reraise(exc_type, value, traceback=None): diff --git a/hy/cmdline.py b/hy/cmdline.py index a9f3af3..f65379d 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -15,13 +15,20 @@ import py_compile import traceback import runpy import types +import time +import linecache +import hashlib +import codeop import astor.code_gen import hy + from hy.lex import hy_parse, mangle +from contextlib import contextmanager from hy.lex.exceptions import PrematureEndOfInput -from hy.compiler import HyASTCompiler, hy_eval, hy_compile, ast_compile +from hy.compiler import (HyASTCompiler, hy_eval, hy_compile, + hy_ast_compile_flags) from hy.errors import (HyLanguageError, HyRequireError, HyMacroExpansionError, filtered_hy_exceptions, hy_exc_handler) from hy.importer import runhy @@ -31,6 +38,11 @@ from hy.models import HyExpression, HyString, HySymbol from hy._compat import builtins, PY3, FileNotFoundError +sys.last_type = None +sys.last_value = None +sys.last_traceback = None + + class HyQuitter(object): def __init__(self, name): self.name = name @@ -51,14 +63,101 @@ class HyQuitter(object): builtins.quit = HyQuitter('quit') builtins.exit = HyQuitter('exit') +@contextmanager +def extend_linecache(add_cmdline_cache): + _linecache_checkcache = linecache.checkcache -class HyCommandCompiler(object): - def __init__(self, module, ast_callback=None, hy_compiler=None): + def _cmdline_checkcache(*args): + _linecache_checkcache(*args) + linecache.cache.update(add_cmdline_cache) + + linecache.checkcache = _cmdline_checkcache + yield + linecache.checkcache = _linecache_checkcache + + +_codeop_maybe_compile = codeop._maybe_compile + + +def _hy_maybe_compile(compiler, source, filename, symbol): + """The `codeop` version of this will compile the same source multiple + times, and, since we have macros and things like `eval-and-compile`, we + can't allow that. + """ + if not isinstance(compiler, HyCompile): + return _codeop_maybe_compile(compiler, source, filename, symbol) + + for line in source.split("\n"): + line = line.strip() + if line and line[0] != ';': + # Leave it alone (could do more with Hy syntax) + break + else: + if symbol != "eval": + # Replace it with a 'pass' statement (i.e. tell the compiler to do + # nothing) + source = "pass" + + return compiler(source, filename, symbol) + + +codeop._maybe_compile = _hy_maybe_compile + + +class HyCompile(codeop.Compile, object): + """This compiler uses `linecache` like + `IPython.core.compilerop.CachingCompiler`. + """ + + def __init__(self, module, locals, ast_callback=None, + hy_compiler=None, cmdline_cache={}): self.module = module + self.locals = locals self.ast_callback = ast_callback self.hy_compiler = hy_compiler + super(HyCompile, self).__init__() + + self.flags |= hy_ast_compile_flags + + self.cmdline_cache = cmdline_cache + + def _cache(self, source, name): + entry = (len(source), + time.time(), + [line + '\n' for line in source.splitlines()], + name) + + linecache.cache[name] = entry + self.cmdline_cache[name] = entry + + def _update_exc_info(self): + self.locals['_hy_last_type'] = sys.last_type + self.locals['_hy_last_value'] = sys.last_value + # Skip our frame. + sys.last_traceback = getattr(sys.last_traceback, 'tb_next', + sys.last_traceback) + self.locals['_hy_last_traceback'] = sys.last_traceback + def __call__(self, source, filename="", symbol="single"): + + if source == 'pass': + # We need to return a no-op to signal that no more input is needed. + return (compile(source, filename, symbol),) * 2 + + hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest() + name = '{}-{}'.format(filename.strip('<>'), hash_digest) + + try: + hy_ast = hy_parse(source, filename=name) + except Exception: + # Capture a traceback without the compiler/REPL frames. + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + self._update_exc_info() + raise + + self._cache(source, name) + try: hy_ast = hy_parse(source, filename=filename) root_ast = ast.Interactive if symbol == 'single' else ast.Module @@ -75,14 +174,39 @@ class HyCommandCompiler(object): if self.ast_callback: self.ast_callback(exec_ast, eval_ast) - exec_code = ast_compile(exec_ast, filename, symbol) - eval_code = ast_compile(eval_ast, filename, 'eval') + exec_code = super(HyCompile, self).__call__(exec_ast, name, symbol) + eval_code = super(HyCompile, self).__call__(eval_ast, name, 'eval') - return exec_code, eval_code - except PrematureEndOfInput: - # Save these so that we can reraise/display when an incomplete - # interactive command is given at the prompt. + except HyLanguageError: + # Hy will raise exceptions during compile-time that Python would + # raise during run-time (e.g. import errors for `require`). In + # order to work gracefully with the Python world, we convert such + # Hy errors to code that purposefully reraises those exceptions in + # the places where Python code expects them. sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + self._update_exc_info() + exec_code = super(HyCompile, self).__call__( + 'import hy._compat; hy._compat.reraise(' + '_hy_last_type, _hy_last_value, _hy_last_traceback)', + name, symbol) + eval_code = super(HyCompile, self).__call__('None', name, 'eval') + + return exec_code, eval_code + + +class HyCommandCompiler(codeop.CommandCompiler, object): + def __init__(self, *args, **kwargs): + self.compiler = HyCompile(*args, **kwargs) + + def __call__(self, *args, **kwargs): + try: + return super(HyCommandCompiler, self).__call__(*args, **kwargs) + except PrematureEndOfInput: + # We have to do this here, because `codeop._maybe_compile` won't + # take `None` for a return value (at least not in Python 2.7) and + # this exception type is also a `SyntaxError`, so it will be caught + # by `code.InteractiveConsole` base methods before it reaches our + # `runsource`. return None @@ -111,11 +235,16 @@ class HyREPL(code.InteractiveConsole, object): self.hy_compiler = HyASTCompiler(self.module) - self.compile = HyCommandCompiler(self.module, self.ast_callback, - self.hy_compiler) + self.cmdline_cache = {} + self.compile = HyCommandCompiler(self.module, + self.locals, + ast_callback=self.ast_callback, + hy_compiler=self.hy_compiler, + cmdline_cache=self.cmdline_cache) self.spy = spy self.last_value = None + self.print_last_value = True if output_fn is None: self.output_fn = repr @@ -133,6 +262,9 @@ class HyREPL(code.InteractiveConsole, object): self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] self.locals.update({sym: None for sym in self._repl_results_symbols}) + # Allow access to the running REPL instance + self.locals['_hy_repl'] = self + def ast_callback(self, exec_ast, eval_ast): if self.spy: try: @@ -146,11 +278,17 @@ class HyREPL(code.InteractiveConsole, object): traceback.format_exc()) self.write(msg) - def _error_wrap(self, error_fn, *args, **kwargs): + def _error_wrap(self, error_fn, exc_info_override=False, *args, **kwargs): sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() - # Sadly, this method in Python 2.7 ignores an overridden - # `sys.excepthook`. + if exc_info_override: + # Use a traceback that doesn't have the REPL frames. + sys.last_type = self.locals.get('_hy_last_type', sys.last_type) + sys.last_value = self.locals.get('_hy_last_value', sys.last_value) + sys.last_traceback = self.locals.get('_hy_last_traceback', + sys.last_traceback) + + # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`. if sys.excepthook is sys.__excepthook__: error_fn(*args, **kwargs) else: @@ -163,6 +301,7 @@ class HyREPL(code.InteractiveConsole, object): filename = self.filename self._error_wrap(super(HyREPL, self).showsyntaxerror, + exc_info_override=True, filename=filename) def showtraceback(self): @@ -289,7 +428,9 @@ def run_repl(hr=None, **kwargs): hr = HyREPL(**kwargs) namespace = hr.locals - with filtered_hy_exceptions(), completion(Completer(namespace)): + with filtered_hy_exceptions(), \ + extend_linecache(hr.cmdline_cache), \ + completion(Completer(namespace)): hr.interact("{appname} {version} using " "{py}({build}) {pyversion} on {os}".format( appname=hy.__appname__, diff --git a/hy/errors.py b/hy/errors.py index 0579e96..a0cd589 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -257,8 +257,7 @@ def hy_exc_filter(exc_type, exc_value, exc_traceback): lines = traceback.format_list(new_tb) - if lines: - lines.insert(0, "Traceback (most recent call last):\n") + lines.insert(0, "Traceback (most recent call last):\n") lines.extend(traceback.format_exception_only(exc_type, exc_value)) output = ''.join(lines) diff --git a/tests/test_bin.py b/tests/test_bin.py index aad45e5..06b3af9 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -6,7 +6,6 @@ import os import re -import sys import shlex import subprocess @@ -523,7 +522,7 @@ def test_bin_hy_tracebacks(): output, error = run_cmd('hy -i "(require not-a-real-module)"') assert output.startswith('=> ') print(error.splitlines()) - req_err(error.splitlines()[3 if PY3 else -3]) + req_err(error.splitlines()[2 if PY3 else -3]) # Modeled after # > python -c 'print("hi' @@ -532,13 +531,14 @@ def test_bin_hy_tracebacks(): # ^ # SyntaxError: EOL while scanning string literal _, error = run_cmd(r'hy -c "(print \""', expect=1) - peoi = ( - ' File "", line 1\n' - ' (print "\n' - ' ^\n' + - '{}PrematureEndOfInput: Partial string literal\n'.format( - 'hy.lex.exceptions.' if PY3 else '')) - assert error == peoi + peoi_re = ( + r'Traceback \(most recent call last\):\n' + r' File "(?:|string-[0-9a-f]+)", line 1\n' + r' \(print "\n' + r' \^\n' + + r'{}PrematureEndOfInput: Partial string literal\n'.format( + r'hy\.lex\.exceptions\.' if PY3 else '')) + assert re.search(peoi_re, error) # Modeled after # > python -i -c "print('" @@ -549,7 +549,7 @@ def test_bin_hy_tracebacks(): # >>> output, error = run_cmd(r'hy -i "(print \""') assert output.startswith('=> ') - assert error.startswith(peoi) + assert re.match(peoi_re, error) # Modeled after # > python -c 'print(a)'