Retain compiled source and file information for exceptions

This commit refactors the exception/error classes and their handling.
It also retains Hy source strings and their originating file information, when
available, all throughout the core parser and compiler functions.

As well, with these changes, calling code is no longer responsible for providing
source and file details to exceptions,

Closes hylang/hy#657.

hy/_compat.py

@@ -6,7 +6,7 @@ try:
     import __builtin__ as builtins
 except ImportError:
     import builtins  # NOQA
-import sys, keyword
+import sys, keyword, textwrap
 
 PY3 = sys.version_info[0] >= 3
 PY35 = sys.version_info >= (3, 5)
@@ -22,11 +22,39 @@ bytes_type   = bytes if PY3 else str          # NOQA
 long_type    = int   if PY3 else long         # NOQA
 string_types = str   if PY3 else basestring   # NOQA
 
+#
+# Inspired by the same-named `six` functions.
+#
 if PY3:
-    exec('def raise_empty(t, *args): raise t(*args) from None')
+    raise_src = textwrap.dedent('''
+    def raise_from(value, from_value):
+        try:
+            raise value from from_value
+        finally:
+            traceback = None
+    ''')
+
+    def reraise(exc_type, value, traceback=None):
+        try:
+            raise value.with_traceback(traceback)
+        finally:
+            traceback = None
+
 else:
-    def raise_empty(t, *args):
+    def raise_from(value, from_value=None):
-        raise t(*args)
+        raise value
+
+    raise_src = textwrap.dedent('''
+    def reraise(exc_type, value, traceback=None):
+        try:
+            raise exc_type, value, traceback
+        finally:
+            traceback = None
+    ''')
+
+raise_code = compile(raise_src, __file__, 'exec')
+exec(raise_code)
+
 
 def isidentifier(x):
     if x in ('True', 'False', 'None', 'print'):

hy/cmdline.py

@@ -19,9 +19,9 @@ import astor.code_gen
 
 import hy
 from hy.lex import hy_parse, mangle
-from hy.lex.exceptions import LexException, PrematureEndOfInput
+from hy.lex.exceptions import PrematureEndOfInput
 from hy.compiler import HyASTCompiler, hy_compile, hy_eval
-from hy.errors import HyTypeError
+from hy.errors import HyTypeError, HyLanguageError, HySyntaxError
 from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
@@ -101,15 +101,11 @@ class HyREPL(code.InteractiveConsole, object):
                 self.showtraceback()
 
         try:
-            try:
+            do = hy_parse(source, filename=filename)
-                do = hy_parse(source)
         except PrematureEndOfInput:
             return True
-        except LexException as e:
+        except HySyntaxError as e:
-            if e.source is None:
+            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
-                e.source = source
-                e.filename = filename
-            error_handler(e, use_simple_traceback=True)
             return False
 
         try:
@@ -121,9 +117,12 @@ class HyREPL(code.InteractiveConsole, object):
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
 
-            value = hy_eval(do, self.locals,
+            value = hy_eval(do, self.locals, self.module,
                             ast_callback=ast_callback,
-                            compiler=self.hy_compiler)
+                            compiler=self.hy_compiler,
+                            filename=filename,
+                            source=source)
+
         except HyTypeError as e:
             if e.source is None:
                 e.source = source
@@ -131,7 +130,7 @@ class HyREPL(code.InteractiveConsole, object):
             error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
             return False
         except Exception as e:
-            error_handler(e)
+            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
             return False
 
         if value is not None:
@@ -208,17 +207,19 @@ SIMPLE_TRACEBACKS = True
 def pretty_error(func, *args, **kw):
     try:
         return func(*args, **kw)
-    except (HyTypeError, LexException) as e:
+    except HyLanguageError as e:
         if SIMPLE_TRACEBACKS:
             print(e, file=sys.stderr)
             sys.exit(1)
         raise
 
 
-def run_command(source):
+def run_command(source, filename=None):
-    tree = hy_parse(source)
+    tree = hy_parse(source, filename=filename)
-    require("hy.cmdline", "__main__", assignments="ALL")
+    __main__ = importlib.import_module('__main__')
-    pretty_error(hy_eval, tree, None, importlib.import_module('__main__'))
+    require("hy.cmdline", __main__, assignments="ALL")
+    pretty_error(hy_eval, tree, None, __main__, filename=filename,
+                 source=source)
     return 0
 
 
@@ -340,7 +341,7 @@ def cmdline_handler(scriptname, argv):
 
     if options.command:
         # User did "hy -c ..."
-        return run_command(options.command)
+        return run_command(options.command, filename='<string>')
 
     if options.mod:
         # User did "hy -m ..."
@@ -356,7 +357,7 @@ def cmdline_handler(scriptname, argv):
     if options.args:
         if options.args[0] == "-":
             # Read the program from stdin
-            return run_command(sys.stdin.read())
+            return run_command(sys.stdin.read(), filename='<stdin>')
 
         else:
             # User did "hy <filename>"
@@ -447,11 +448,12 @@ def hy2py_main():
 
     if options.FILE is None or options.FILE == '-':
         source = sys.stdin.read()
+        hst = pretty_error(hy_parse, source, filename='<stdin>')
     else:
         with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
             source = source_file.read()
+            hst = hy_parse(source, filename=options.FILE)
 
-    hst = pretty_error(hy_parse, source)
     if options.with_source:
         # need special printing on Windows in case the
         # codepage doesn't support utf-8 characters

hy/compiler.py

@@ -9,20 +9,21 @@ from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
 from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole,
                                notpexpr, dolike, pexpr, times, Tag, tag, unpack)
 from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
-from hy.errors import HyCompileError, HyTypeError
+from hy.errors import (HyCompileError, HyTypeError, HyEvalError,
+                       HyInternalError)
 
 from hy.lex import mangle, unmangle
 
-from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
+from hy._compat import (string_types, str_type, bytes_type, long_type, PY3,
-                        PY35, raise_empty)
+                        PY35, reraise)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
 import hy.core
 
+import pkgutil
 import traceback
 import importlib
 import inspect
-import pkgutil
 import types
 import ast
 import sys
@@ -340,22 +341,33 @@ def is_unpack(kind, x):
 class HyASTCompiler(object):
     """A Hy-to-Python AST compiler"""
 
-    def __init__(self, module):
+    def __init__(self, module, filename=None, source=None):
         """
         Parameters
         ----------
         module: str or types.ModuleType
-            Module in which the Hy tree is evaluated.
+            Module name or object in which the Hy tree is evaluated.
+        filename: str, optional
+            The name of the file for the source to be compiled.
+            This is optional information for informative error messages and
+            debugging.
+        source: str, optional
+            The source for the file, if any, being compiled.  This is optional
+            information for informative error messages and debugging.
         """
         self.anon_var_count = 0
         self.imports = defaultdict(set)
         self.temp_if = None
 
         if not inspect.ismodule(module):
-            module = importlib.import_module(module)
+            self.module = importlib.import_module(module)
-
+        else:
             self.module = module
-        self.module_name = module.__name__
+
+        self.module_name = self.module.__name__
+
+        self.filename = filename
+        self.source = source
 
         # Hy expects these to be present, so we prep the module for Hy
         # compilation.
@@ -431,13 +443,15 @@ class HyASTCompiler(object):
             # nested; so let's re-raise this exception, let's not wrap it in
             # another HyCompileError!
             raise
-        except HyTypeError:
+        except HyTypeError as e:
-            raise
+            reraise(type(e), e, None)
         except Exception as e:
-            raise_empty(HyCompileError, e, sys.exc_info()[2])
+            f_exc = traceback.format_exc()
+            exc_msg = "Internal Compiler Bug 😱\n⤷ {}".format(f_exc)
+            reraise(HyCompileError, HyCompileError(exc_msg), sys.exc_info()[2])
 
     def _syntax_error(self, expr, message):
-        return HyTypeError(expr, message)
+        return HyTypeError(message, self.filename, expr, self.source)
 
     def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
                          oldpy_unpack=False):
@@ -1614,7 +1628,29 @@ class HyASTCompiler(object):
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
 
-        hy_eval(new_expr + body, self.module.__dict__, self.module)
+        try:
+            hy_eval(new_expr + body,
+                    self.module.__dict__,
+                    self.module,
+                    filename=self.filename,
+                    source=self.source)
+        except HyInternalError:
+            # Unexpected "meta" compilation errors need to be treated
+            # like normal (unexpected) compilation errors at this level
+            # (or the compilation level preceding this one).
+            raise
+        except Exception as e:
+            # These could be expected Hy language errors (e.g. syntax errors)
+            # or regular Python runtime errors that do not signify errors in
+            # the compilation *process* (although compilation did technically
+            # fail).
+            # We wrap these exceptions and pass them through.
+            reraise(HyEvalError,
+                    HyEvalError(str(e),
+                                self.filename,
+                                body,
+                                self.source),
+                    sys.exc_info()[2])
 
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
@@ -1798,8 +1834,13 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False):
 
 
 def hy_eval(hytree, locals=None, module=None, ast_callback=None,
-            compiler=None):
+            compiler=None, filename='<string>', source=None):
     """Evaluates a quoted expression and returns the value.
+
+    If you're evaluating hand-crafted AST trees, make sure the line numbers
+    are set properly.  Try `fix_missing_locations` and related functions in the
+    Python `ast` library.
+
     Examples
     --------
        => (eval '(print "Hello World"))
@@ -1812,8 +1853,8 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None,
 
     Parameters
     ----------
-    hytree: a Hy expression tree
+    hytree: HyObject
-        Source code to parse.
+        The Hy AST object to evaluate.
 
     locals: dict, optional
         Local environment in which to evaluate the Hy tree.  Defaults to the
@@ -1835,6 +1876,19 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None,
         An existing Hy compiler to use for compilation.  Also serves as
         the `module` value when given.
 
+    filename: str, optional
+        The filename corresponding to the source for `tree`.  This will be
+        overridden by the `filename` field of `tree`, if any; otherwise, it
+        defaults to "<string>".  When `compiler` is given, its `filename` field
+        value is always used.
+
+    source: str, optional
+        A string containing the source code for `tree`.  This will be
+        overridden by the `source` field of `tree`, if any; otherwise,
+        if `None`, an attempt will be made to obtain it from the module given by
+        `module`.  When `compiler` is given, its `source` field value is always
+        used.
+
     Returns
     -------
     out : Result of evaluating the Hy compiled tree.
@@ -1849,36 +1903,53 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None,
     if not isinstance(locals, dict):
         raise TypeError("Locals must be a dictionary")
 
-    _ast, expr = hy_compile(hytree, module=module, get_expr=True,
+    # Does the Hy AST object come with its own information?
-                            compiler=compiler)
+    filename = getattr(hytree, 'filename', filename) or '<string>'
+    source = getattr(hytree, 'source', source)
 
-    # Spoof the positions in the generated ast...
+    _ast, expr = hy_compile(hytree, module, get_expr=True,
-    for node in ast.walk(_ast):
+                            compiler=compiler, filename=filename,
-        node.lineno = 1
+                            source=source)
-        node.col_offset = 1
-
-    for node in ast.walk(expr):
-        node.lineno = 1
-        node.col_offset = 1
 
     if ast_callback:
         ast_callback(_ast, expr)
 
-    globals = module.__dict__
-
     # Two-step eval: eval() the body of the exec call
-    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
+    eval(ast_compile(_ast, filename, "exec"),
+         module.__dict__, locals)
 
     # Then eval the expression context and return that
-    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
+    return eval(ast_compile(expr, filename, "eval"),
+                module.__dict__, locals)
 
 
-def hy_compile(tree, module=None, root=ast.Module, get_expr=False,
+def _module_file_source(module_name, filename, source):
-               compiler=None):
+    """Try to obtain missing filename and source information from a module name
-    """Compile a Hy tree into a Python AST tree.
+    without actually loading the module.
+    """
+    if filename is None or source is None:
+        mod_loader = pkgutil.get_loader(module_name)
+        if mod_loader:
+            if filename is None:
+                filename = mod_loader.get_filename(module_name)
+            if source is None:
+                source = mod_loader.get_source(module_name)
+
+    # We need a non-None filename.
+    filename = filename or '<string>'
+
+    return filename, source
+
+
+def hy_compile(tree, module, root=ast.Module, get_expr=False,
+               compiler=None, filename=None, source=None):
+    """Compile a HyObject tree into a Python AST Module.
 
     Parameters
     ----------
+    tree: HyObject
+        The Hy AST object to compile.
+
     module: str or types.ModuleType, optional
         Module, or name of the module, in which the Hy tree is evaluated.
         The module associated with `compiler` takes priority over this value.
@@ -1893,18 +1964,43 @@ def hy_compile(tree, module=None, root=ast.Module, get_expr=False,
         An existing Hy compiler to use for compilation.  Also serves as
         the `module` value when given.
 
+    filename: str, optional
+        The filename corresponding to the source for `tree`.  This will be
+        overridden by the `filename` field of `tree`, if any; otherwise, it
+        defaults to "<string>".  When `compiler` is given, its `filename` field
+        value is always used.
+
+    source: str, optional
+        A string containing the source code for `tree`.  This will be
+        overridden by the `source` field of `tree`, if any; otherwise,
+        if `None`, an attempt will be made to obtain it from the module given by
+        `module`.  When `compiler` is given, its `source` field value is always
+        used.
+
     Returns
     -------
     out : A Python AST tree
     """
     module = get_compiler_module(module, compiler, False)
 
+    if isinstance(module, string_types):
+        if module.startswith('<') and module.endswith('>'):
+            module = types.ModuleType(module)
+        else:
+            module = importlib.import_module(ast_str(module, piecewise=True))
+
+    if not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    filename = getattr(tree, 'filename', filename)
+    source = getattr(tree, 'source', source)
+
     tree = wrap_value(tree)
     if not isinstance(tree, HyObject):
-        raise HyCompileError("`tree` must be a HyObject or capable of "
+        raise TypeError("`tree` must be a HyObject or capable of "
                         "being promoted to one")
 
-    compiler = compiler or HyASTCompiler(module)
+    compiler = compiler or HyASTCompiler(module, filename=filename, source=source)
     result = compiler.compile(tree)
     expr = result.force_expr
 

hy/core/bootstrap.hy

@@ -14,15 +14,14 @@
      (if* (not (isinstance macro-name hy.models.HySymbol))
           (raise
             (hy.errors.HyTypeError
-              macro-name
               (% "received a `%s' instead of a symbol for macro name"
-                 (. (type name)
+                 (. (type name) --name--))
-                    __name__)))))
+              --file-- macro-name None)))
      (for [kw '[&kwonly &kwargs]]
        (if* (in kw lambda-list)
-            (raise (hy.errors.HyTypeError macro-name
+            (raise (hy.errors.HyTypeError (% "macros cannot use %s"
-                                          (% "macros cannot use %s"
+                                             kw)
-                                             kw)))))
+                                          --file-- macro-name None))))
      ;; this looks familiar...
      `(eval-and-compile
         (import hy)
@@ -45,9 +44,9 @@
   (if (and (not (isinstance tag-name hy.models.HySymbol))
            (not (isinstance tag-name hy.models.HyString)))
       (raise (hy.errors.HyTypeError
-               tag-name
                (% "received a `%s' instead of a symbol for tag macro name"
-                  (. (type tag-name) __name__)))))
+                  (. (type tag-name) --name--))
+               --file-- tag-name None)))
   (if (or (= tag-name ":")
           (= tag-name "&"))
       (raise (NameError (% "%s can't be used as a tag macro name" tag-name))))
@@ -58,9 +57,8 @@
      ((hy.macros.tag ~tag-name)
       (fn ~lambda-list ~@body))))
 
-(defmacro macro-error [location reason]
+(defmacro macro-error [expression reason &optional [filename '--name--]]
-  "Error out properly within a macro at `location` giving `reason`."
+  `(raise (hy.errors.HyMacroExpansionError ~reason ~filename ~expression None)))
-  `(raise (hy.errors.HyMacroExpansionError ~location ~reason)))
 
 (defmacro defn [name lambda-list &rest body]
   "Define `name` as a function with `lambda-list` signature and body `body`."

hy/errors.py

@@ -5,41 +5,65 @@
 
 import traceback
 
+from functools import reduce
+
 from clint.textui import colored
 
 
 class HyError(Exception):
-    """
+    def __init__(self, message, *args):
-    Generic Hy error. All internal Exceptions will be subclassed from this
-    Exception.
-    """
-    pass
-
-
-class HyCompileError(HyError):
-    def __init__(self, exception, traceback=None):
-        self.exception = exception
-        self.traceback = traceback
-
-    def __str__(self):
-        if isinstance(self.exception, HyTypeError):
-            return str(self.exception)
-        if self.traceback:
-            tb = "".join(traceback.format_tb(self.traceback)).strip()
-        else:
-            tb = "No traceback available. 😟"
-        return("Internal Compiler Bug 😱\n⤷ %s: %s\nCompilation traceback:\n%s"
-               % (self.exception.__class__.__name__,
-                  self.exception, tb))
-
-
-class HyTypeError(TypeError):
-    def __init__(self, expression, message):
-        super(HyTypeError, self).__init__(message)
-        self.expression = expression
         self.message = message
-        self.source = None
+        super(HyError, self).__init__(message, *args)
-        self.filename = None
+
+
+class HyInternalError(HyError):
+    """Unexpected errors occurring during compilation or parsing of Hy code.
+
+    Errors sub-classing this are not intended to be user-facing, and will,
+    hopefully, never be seen by users!
+    """
+
+    def __init__(self, message, *args):
+        super(HyInternalError, self).__init__(message, *args)
+
+
+class HyLanguageError(HyError):
+    """Errors caused by invalid use of the Hy language.
+
+    This, and any errors inheriting from this, are user-facing.
+    """
+
+    def __init__(self, message, *args):
+        super(HyLanguageError, self).__init__(message, *args)
+
+
+class HyCompileError(HyInternalError):
+    """Unexpected errors occurring within the compiler."""
+
+
+class HyTypeError(HyLanguageError, TypeError):
+    """TypeErrors occurring during the normal use of Hy."""
+
+    def __init__(self, message, filename=None, expression=None, source=None):
+        """
+        Parameters
+        ----------
+        message: str
+            The message to display for this error.
+        filename: str, optional
+            The filename for the source code generating this error.
+        expression: HyObject, optional
+            The Hy expression generating this error.
+        source: str, optional
+            The actual source code generating this error.
+        """
+        self.message = message
+        self.filename = filename
+        self.expression = expression
+        self.source = source
+
+        super(HyTypeError, self).__init__(message, filename, expression,
+                                          source)
 
     def __str__(self):
 
@@ -93,12 +117,92 @@ class HyTypeError(TypeError):
 
 
 class HyMacroExpansionError(HyTypeError):
-    pass
+    """Errors caused by invalid use of Hy macros.
+
+    This, and any errors inheriting from this, are user-facing.
+    """
 
 
-class HyIOError(HyError, IOError):
+class HyEvalError(HyLanguageError):
+    """Errors occurring during code evaluation at compile-time.
+
+    These errors distinguish unexpected errors within the compilation process
+    (i.e. `HyInternalError`s) from unrelated errors in user code evaluated by
+    the compiler (e.g. in `eval-and-compile`).
+
+    This, and any errors inheriting from this, are user-facing.
     """
-    Trivial subclass of IOError and HyError, to distinguish between
+
-    IOErrors raised by Hy itself as opposed to Hy programs.
+
+class HyIOError(HyInternalError, IOError):
+    """ Subclass used to distinguish between IOErrors raised by Hy itself as
+    opposed to Hy programs.
     """
-    pass
+
+
+class HySyntaxError(HyLanguageError, SyntaxError):
+    """Error during the Lexing of a Hython expression."""
+
+    def __init__(self, message, filename=None, lineno=-1, colno=-1,
+                 source=None):
+        """
+        Parameters
+        ----------
+        message: str
+            The exception's message.
+        filename: str, optional
+            The filename for the source code generating this error.
+        lineno: int, optional
+            The line number of the error.
+        colno: int, optional
+            The column number of the error.
+        source: str, optional
+            The actual source code generating this error.
+        """
+        self.message = message
+        self.filename = filename
+        self.lineno = lineno
+        self.colno = colno
+        self.source = source
+        super(HySyntaxError, self).__init__(message,
+                                            # The builtin `SyntaxError` needs a
+                                            # tuple.
+                                            (filename, lineno, colno, source))
+
+    @staticmethod
+    def from_expression(message, expression, filename=None, source=None):
+        if not source:
+            # Maybe the expression object has its own source.
+            source = getattr(expression, 'source', None)
+
+        if not filename:
+            filename = getattr(expression, 'filename', None)
+
+        if source:
+            lineno = expression.start_line
+            colno = expression.start_column
+            end_line = getattr(expression, 'end_line', len(source))
+            lines = source.splitlines()
+            source = '\n'.join(lines[lineno-1:end_line])
+        else:
+            # We could attempt to extract the source given a filename, but we
+            # don't.
+            lineno = colno = -1
+
+        return HySyntaxError(message, filename, lineno, colno, source)
+
+    def __str__(self):
+
+        output = traceback.format_exception_only(SyntaxError, self)
+
+        output[-1] = colored.yellow(output[-1])
+        if len(self.source) > 0:
+            output[-2] = colored.green(output[-2])
+            for line in output[::-2]:
+                if line.strip().startswith(
+                        'File "{}", line'.format(self.filename)):
+                    break
+            output[-3] = colored.red(output[-3])
+
+        # Avoid "...expected str instance, ColoredString found"
+        return reduce(lambda x, y: x + y, output)

hy/importer.py

@@ -17,10 +17,8 @@ import importlib
 from functools import partial
 from contextlib import contextmanager
 
-from hy.errors import HyTypeError
 from hy.compiler import hy_compile, hy_ast_compile_flags
 from hy.lex import hy_parse
-from hy.lex.exceptions import LexException
 from hy._compat import PY3
 
 
@@ -153,15 +151,9 @@ if PY3:
     def _hy_source_to_code(self, data, path, _optimize=-1):
         if _could_be_hy_src(path):
             source = data.decode("utf-8")
-            try:
+            hy_tree = hy_parse(source, filename=path)
-                hy_tree = hy_parse(source)
             with loader_module_obj(self) as module:
                 data = hy_compile(hy_tree, module)
-            except (HyTypeError, LexException) as e:
-                if e.source is None:
-                    e.source = source
-                    e.filename = path
-                raise
 
         return _py_source_to_code(self, data, path, _optimize=_optimize)
 
@@ -287,19 +279,15 @@ else:
             fullname = self._fix_name(fullname)
             if fullname is None:
                 fullname = self.fullname
-            try:
+
             hy_source = self.get_source(fullname)
-                hy_tree = hy_parse(hy_source)
+            hy_tree = hy_parse(hy_source, filename=self.filename)
+
             with loader_module_obj(self) as module:
                 hy_ast = hy_compile(hy_tree, module)
 
             code = compile(hy_ast, self.filename, 'exec',
                            hy_ast_compile_flags)
-            except (HyTypeError, LexException) as e:
-                if e.source is None:
-                    e.source = hy_source
-                    e.filename = self.filename
-                raise
 
             if not sys.dont_write_bytecode:
                 try:
@@ -453,7 +441,7 @@ else:
             try:
                 flags = None
                 if _could_be_hy_src(filename):
-                    hy_tree = hy_parse(source_str)
+                    hy_tree = hy_parse(source_str, filename=filename)
 
                     if module is None:
                         module = inspect.getmodule(inspect.stack()[1][0])
@@ -465,9 +453,6 @@ else:
 
                 codeobject = compile(source, dfile or filename, 'exec', flags)
             except Exception as err:
-                if isinstance(err, (HyTypeError, LexException)) and err.source is None:
-                    err.source = source_str
-                    err.filename = filename
 
                 py_exc = py_compile.PyCompileError(err.__class__, err,
                                                    dfile or filename)

hy/lex/__init__.py

@@ -8,9 +8,10 @@ import re
 import sys
 import unicodedata
 
-from hy._compat import str_type, isidentifier, UCS4
+from hy._compat import str_type, isidentifier, UCS4, reraise
 from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
+from hy.errors import HySyntaxError
 
 try:
     from io import StringIO
@@ -18,7 +19,7 @@ except ImportError:
     from StringIO import StringIO
 
 
-def hy_parse(source):
+def hy_parse(source, filename='<string>'):
     """Parse a Hy source string.
 
     Parameters
@@ -26,31 +27,51 @@ def hy_parse(source):
     source: string
         Source code to parse.
 
+    filename: string, optional
+        File name corresponding to source.  Defaults to "<string>".
+
     Returns
     -------
-    out : instance of `types.CodeType`
+    out : HyExpression
     """
-    source = re.sub(r'\A#!.*', '', source)
+    _source = re.sub(r'\A#!.*', '', source)
-    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
+    try:
+        res = HyExpression([HySymbol("do")] +
+                           tokenize(_source + "\n",
+                                    filename=filename))
+        res.source = source
+        res.filename = filename
+        return res
+    except HySyntaxError as e:
+        reraise(type(e), e, None)
 
 
-def tokenize(buf):
+class ParserState(object):
-    """
+    def __init__(self, source, filename):
-    Tokenize a Lisp file or string buffer into internal Hy objects.
+        self.source = source
+        self.filename = filename
+
+
+def tokenize(source, filename=None):
+    """ Tokenize a Lisp file or string buffer into internal Hy objects.
+
+    Parameters
+    ----------
+    source: str
+        The source to tokenize.
+    filename: str, optional
+        The filename corresponding to `source`.
     """
     from hy.lex.lexer import lexer
     from hy.lex.parser import parser
     from rply.errors import LexingError
     try:
-        return parser.parse(lexer.lex(buf))
+        return parser.parse(lexer.lex(source),
+                            state=ParserState(source, filename))
     except LexingError as e:
         pos = e.getsourcepos()
-        raise LexException("Could not identify the next token.",
+        raise LexException("Could not identify the next token.", filename,
-                           pos.lineno, pos.colno, buf)
+                           pos.lineno, pos.colno, source)
-    except LexException as e:
-        if e.source is None:
-            e.source = buf
-        raise
 
 
 mangle_delim = 'X'

hy/lex/exceptions.py

@@ -1,49 +1,34 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
-
+from hy.errors import HySyntaxError
-from hy.errors import HyError
 
 
-class LexException(HyError):
+class LexException(HySyntaxError):
-    """Error during the Lexing of a Hython expression."""
-    def __init__(self, message, lineno, colno, source=None):
-        super(LexException, self).__init__(message)
-        self.message = message
-        self.lineno = lineno
-        self.colno = colno
-        self.source = source
-        self.filename = '<stdin>'
 
-    def __str__(self):
+    @classmethod
-        from hy.errors import colored
+    def from_lexer(cls, message, state, token):
-
+        source_pos = token.getsourcepos()
-        line = self.lineno
+        if token.source_pos:
-        start = self.colno
+            lineno = source_pos.lineno
-
+            colno = source_pos.colno
-        result = ""
-
-        source = self.source.split("\n")
-
-        if line > 0 and start > 0:
-            result += '  File "%s", line %d, column %d\n\n' % (self.filename,
-                                                               line,
-                                                               start)
-
-            if len(self.source) > 0:
-                source_line = source[line-1]
         else:
-                source_line = ""
+            lineno = -1
+            colno = -1
 
-            result += '  %s\n' % colored.red(source_line)
+        if state.source:
-            result += '  %s%s\n' % (' '*(start-1), colored.green('^'))
+            lines = state.source.splitlines()
+            if lines[-1] == '':
+                del lines[-1]
 
-        result += colored.yellow("LexException: %s\n\n" % self.message)
+            if lineno < 1:
+                lineno = len(lines)
+            if colno < 1:
+                colno = len(lines[-1])
 
-        return result
+            source = lines[lineno - 1]
+        return cls(message, state.filename, lineno, colno, source)
 
 
 class PrematureEndOfInput(LexException):
-    """We got a premature end of input"""
+    pass
-    def __init__(self, message):
-        super(PrematureEndOfInput, self).__init__(message, -1, -1)

hy/lex/parser.py

@@ -22,10 +22,10 @@ pg = ParserGenerator([rule.name for rule in lexer.rules] + ['$end'])
 
 def set_boundaries(fun):
     @wraps(fun)
-    def wrapped(p):
+    def wrapped(state, p):
         start = p[0].source_pos
         end = p[-1].source_pos
-        ret = fun(p)
+        ret = fun(state, p)
         ret.start_line = start.lineno
         ret.start_column = start.colno
         if start is not end:
@@ -40,9 +40,9 @@ def set_boundaries(fun):
 
 def set_quote_boundaries(fun):
     @wraps(fun)
-    def wrapped(p):
+    def wrapped(state, p):
         start = p[0].source_pos
-        ret = fun(p)
+        ret = fun(state, p)
         ret.start_line = start.lineno
         ret.start_column = start.colno
         ret.end_line = p[-1].end_line
@@ -52,54 +52,45 @@ def set_quote_boundaries(fun):
 
 
 @pg.production("main : list_contents")
-def main(p):
+def main(state, p):
     return p[0]
 
 
 @pg.production("main : $end")
-def main_empty(p):
+def main_empty(state, p):
     return []
 
 
-def reject_spurious_dots(*items):
-    "Reject the spurious dots from items"
-    for list in items:
-        for tok in list:
-            if tok == "." and type(tok) == HySymbol:
-                raise LexException("Malformed dotted list",
-                                   tok.start_line, tok.start_column)
-
-
 @pg.production("paren : LPAREN list_contents RPAREN")
 @set_boundaries
-def paren(p):
+def paren(state, p):
     return HyExpression(p[1])
 
 
 @pg.production("paren : LPAREN RPAREN")
 @set_boundaries
-def empty_paren(p):
+def empty_paren(state, p):
     return HyExpression([])
 
 
 @pg.production("list_contents : term list_contents")
-def list_contents(p):
+def list_contents(state, p):
     return [p[0]] + p[1]
 
 
 @pg.production("list_contents : term")
-def list_contents_single(p):
+def list_contents_single(state, p):
     return [p[0]]
 
 
 @pg.production("list_contents : DISCARD term discarded_list_contents")
-def list_contents_empty(p):
+def list_contents_empty(state, p):
     return []
 
 
 @pg.production("discarded_list_contents : DISCARD term discarded_list_contents")
 @pg.production("discarded_list_contents :")
-def discarded_list_contents(p):
+def discarded_list_contents(state, p):
     pass
 
 
@@ -109,58 +100,58 @@ def discarded_list_contents(p):
 @pg.production("term : list")
 @pg.production("term : set")
 @pg.production("term : string")
-def term(p):
+def term(state, p):
     return p[0]
 
 
 @pg.production("term : DISCARD term term")
-def term_discard(p):
+def term_discard(state, p):
     return p[2]
 
 
 @pg.production("term : QUOTE term")
 @set_quote_boundaries
-def term_quote(p):
+def term_quote(state, p):
     return HyExpression([HySymbol("quote"), p[1]])
 
 
 @pg.production("term : QUASIQUOTE term")
 @set_quote_boundaries
-def term_quasiquote(p):
+def term_quasiquote(state, p):
     return HyExpression([HySymbol("quasiquote"), p[1]])
 
 
 @pg.production("term : UNQUOTE term")
 @set_quote_boundaries
-def term_unquote(p):
+def term_unquote(state, p):
     return HyExpression([HySymbol("unquote"), p[1]])
 
 
 @pg.production("term : UNQUOTESPLICE term")
 @set_quote_boundaries
-def term_unquote_splice(p):
+def term_unquote_splice(state, p):
     return HyExpression([HySymbol("unquote-splice"), p[1]])
 
 
 @pg.production("term : HASHSTARS term")
 @set_quote_boundaries
-def term_hashstars(p):
+def term_hashstars(state, p):
     n_stars = len(p[0].getstr()[1:])
     if n_stars == 1:
         sym = "unpack-iterable"
     elif n_stars == 2:
         sym = "unpack-mapping"
     else:
-        raise LexException(
+        raise LexException.from_lexer(
             "Too many stars in `#*` construct (if you want to unpack a symbol "
             "beginning with a star, separate it with whitespace)",
-            p[0].source_pos.lineno, p[0].source_pos.colno)
+            state, p[0])
     return HyExpression([HySymbol(sym), p[1]])
 
 
 @pg.production("term : HASHOTHER term")
 @set_quote_boundaries
-def hash_other(p):
+def hash_other(state, p):
     # p == [(Token('HASHOTHER', '#foo'), bar)]
     st = p[0].getstr()[1:]
     str_object = HyString(st)
@@ -170,63 +161,63 @@ def hash_other(p):
 
 @pg.production("set : HLCURLY list_contents RCURLY")
 @set_boundaries
-def t_set(p):
+def t_set(state, p):
     return HySet(p[1])
 
 
 @pg.production("set : HLCURLY RCURLY")
 @set_boundaries
-def empty_set(p):
+def empty_set(state, p):
     return HySet([])
 
 
 @pg.production("dict : LCURLY list_contents RCURLY")
 @set_boundaries
-def t_dict(p):
+def t_dict(state, p):
     return HyDict(p[1])
 
 
 @pg.production("dict : LCURLY RCURLY")
 @set_boundaries
-def empty_dict(p):
+def empty_dict(state, p):
     return HyDict([])
 
 
 @pg.production("list : LBRACKET list_contents RBRACKET")
 @set_boundaries
-def t_list(p):
+def t_list(state, p):
     return HyList(p[1])
 
 
 @pg.production("list : LBRACKET RBRACKET")
 @set_boundaries
-def t_empty_list(p):
+def t_empty_list(state, p):
     return HyList([])
 
 
 @pg.production("string : STRING")
 @set_boundaries
-def t_string(p):
+def t_string(state, p):
     # Replace the single double quotes with triple double quotes to allow
     # embedded newlines.
     try:
         s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
     except SyntaxError:
-        raise LexException("Can't convert {} to a HyString".format(p[0].value),
+        raise LexException.from_lexer("Can't convert {} to a HyString".format(p[0].value),
-            p[0].source_pos.lineno, p[0].source_pos.colno)
+                                      state, p[0])
     return (HyString if isinstance(s, str_type) else HyBytes)(s)
 
 
 @pg.production("string : PARTIAL_STRING")
-def t_partial_string(p):
+def t_partial_string(state, p):
     # Any unterminated string requires more input
-    raise PrematureEndOfInput("Premature end of input")
+    raise PrematureEndOfInput.from_lexer("Partial string literal", state, p[0])
 
 
 bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING')
 @pg.production("string : BRACKETSTRING")
 @set_boundaries
-def t_bracket_string(p):
+def t_bracket_string(state, p):
     m = bracket_string_re.match(p[0].value)
     delim, content = m.groups()
     return HyString(content, brackets=delim)
@@ -234,7 +225,7 @@ def t_bracket_string(p):
 
 @pg.production("identifier : IDENTIFIER")
 @set_boundaries
-def t_identifier(p):
+def t_identifier(state, p):
     obj = p[0].value
 
     val = symbol_like(obj)
@@ -243,11 +234,11 @@ def t_identifier(p):
 
     if "." in obj and symbol_like(obj.split(".", 1)[0]) is not None:
         # E.g., `5.attr` or `:foo.attr`
-        raise LexException(
+        raise LexException.from_lexer(
             'Cannot access attribute on anything other than a name (in '
             'order to get attributes of expressions, use '
             '`(. <expression> <attr>)` or `(.<attr> <expression>)`)',
-            p[0].source_pos.lineno, p[0].source_pos.colno)
+            state, p[0])
 
     return HySymbol(obj)
 
@@ -284,14 +275,24 @@ def symbol_like(obj):
 
 
 @pg.error
-def error_handler(token):
+def error_handler(state, token):
     tokentype = token.gettokentype()
     if tokentype == '$end':
-        raise PrematureEndOfInput("Premature end of input")
+        source_pos = token.source_pos or token.getsourcepos()
+        source = state.source
+        if source_pos:
+            lineno = source_pos.lineno
+            colno = source_pos.colno
         else:
-        raise LexException(
+            lineno = -1
-            "Ran into a %s where it wasn't expected." % tokentype,
+            colno = -1
-            token.source_pos.lineno, token.source_pos.colno)
+
+        raise PrematureEndOfInput.from_lexer("Premature end of input", state,
+                                             token)
+    else:
+        raise LexException.from_lexer(
+            "Ran into a %s where it wasn't expected." % tokentype, state,
+            token)
 
 
 parser = pg.build()

hy/macros.py

@@ -1,14 +1,16 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
+import sys
 import importlib
 import inspect
 import pkgutil
 
-from hy._compat import PY3, string_types
+from contextlib import contextmanager
+
+from hy._compat import PY3, string_types, reraise
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
-
 from hy.errors import HyTypeError, HyMacroExpansionError
 
 try:
@@ -257,6 +259,32 @@ def make_empty_fn_copy(fn):
     return empty_fn
 
 
+@contextmanager
+def macro_exceptions(module, macro_tree, compiler=None):
+    try:
+        yield
+    except Exception as e:
+        try:
+            filename = inspect.getsourcefile(module)
+            source = inspect.getsource(module)
+        except TypeError:
+            if compiler:
+                filename = compiler.filename
+                source = compiler.source
+
+        if not isinstance(e, HyTypeError):
+            exc_type = HyMacroExpansionError
+            msg = "expanding `{}': ".format(macro_tree[0])
+            msg += str(e).replace("<lambda>()", "", 1).strip()
+        else:
+            exc_type = HyTypeError
+            msg = e.message
+
+        reraise(exc_type,
+                exc_type(msg, filename, macro_tree, source),
+                sys.exc_info()[2].tb_next)
+
+
 def macroexpand(tree, module, compiler=None, once=False):
     """Expand the toplevel macros for the given Hy AST tree.
 
@@ -324,23 +352,10 @@ def macroexpand(tree, module, compiler=None, once=False):
                 compiler = HyASTCompiler(module)
             opts['compiler'] = compiler
 
-        try:
+        with macro_exceptions(module, tree, compiler):
             m_copy = make_empty_fn_copy(m)
             m_copy(module.__name__, *tree[1:], **opts)
-        except TypeError as e:
-            msg = "expanding `" + str(tree[0]) + "': "
-            msg += str(e).replace("<lambda>()", "", 1).strip()
-            raise HyMacroExpansionError(tree, msg)
-
-        try:
             obj = m(module.__name__, *tree[1:], **opts)
-        except HyTypeError as e:
-            if e.expression is None:
-                e.expression = tree
-            raise
-        except Exception as e:
-            msg = "expanding `" + str(tree[0]) + "': " + repr(e)
-            raise HyMacroExpansionError(tree, msg)
 
         if isinstance(obj, HyExpression):
             obj.module = inspect.getmodule(m)
@@ -375,7 +390,8 @@ def tag_macroexpand(tag, tree, module):
                      None)
 
     if tag_macro is None:
-        raise HyTypeError(tag, "'{0}' is not a defined tag macro.".format(tag))
+        raise HyTypeError("`{0}' is not a defined tag macro.".format(tag),
+                          None, tag, None)
 
     expr = tag_macro(tree)
 

tests/compilers/test_ast.py

@@ -10,7 +10,7 @@ from hy.models import HyObject
 from hy.compiler import hy_compile, hy_eval
 from hy.errors import HyCompileError, HyTypeError
 from hy.lex import hy_parse
-from hy.lex.exceptions import LexException
+from hy.lex.exceptions import LexException, PrematureEndOfInput
 from hy._compat import PY3
 
 import ast
@@ -474,7 +474,7 @@ def test_lambda_list_keywords_kwonly():
     else:
         exception = cant_compile(kwonly_demo)
         assert isinstance(exception, HyTypeError)
-        message, = exception.args
+        message = exception.args[0]
         assert message == "&kwonly parameters require Python 3"
 
 
@@ -547,7 +547,7 @@ def test_compile_error():
 
 def test_for_compile_error():
     """Ensure we get compile error in tricky 'for' cases"""
-    with pytest.raises(LexException) as excinfo:
+    with pytest.raises(PrematureEndOfInput) as excinfo:
         can_compile("(fn [] (for)")
     assert excinfo.value.message == "Premature end of input"
 

tests/importer/test_importer.py

@@ -14,10 +14,10 @@ from fractions import Fraction
 import pytest
 
 import hy
-from hy.errors import HyTypeError
 from hy.lex import hy_parse
-from hy.lex.exceptions import LexException
+from hy.errors import HyLanguageError
-from hy.compiler import hy_compile
+from hy.lex.exceptions import PrematureEndOfInput
+from hy.compiler import hy_eval, hy_compile
 from hy.importer import HyLoader, cache_from_source
 
 try:
@@ -57,7 +57,7 @@ def test_runpy():
 
 
 def test_stringer():
-    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '__main__')
+    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), __name__)
 
     assert type(_ast.body[0]) == ast.FunctionDef
 
@@ -79,14 +79,8 @@ def test_imports():
 def test_import_error_reporting():
     "Make sure that (import) reports errors correctly."
 
-    def _import_error_test():
+    with pytest.raises(HyLanguageError):
-        try:
+        hy_compile(hy_parse("(import \"sys\")"), __name__)
-            _ = hy_compile(hy_parse("(import \"sys\")"), '__main__')
-        except HyTypeError:
-            return "Error reported"
-
-    assert _import_error_test() == "Error reported"
-    assert _import_error_test() is not None
 
 
 def test_import_error_cleanup():
@@ -124,7 +118,7 @@ def test_import_autocompiles():
 
 def test_eval():
     def eval_str(s):
-        return hy.eval(hy.read_str(s))
+        return hy_eval(hy.read_str(s), filename='<string>', source=s)
 
     assert eval_str('[1 2 3]') == [1, 2, 3]
     assert eval_str('{"dog" "bark" "cat" "meow"}') == {
@@ -205,8 +199,7 @@ def test_reload():
         assert mod.a == 11
         assert mod.b == 20
 
-        # Now cause a `LexException`, and confirm that the good module and its
+        # Now cause a syntax error
-        # contents stick around.
         unlink(source)
 
         with open(source, "w") as f:
@@ -214,7 +207,7 @@ def test_reload():
             f.write("(setv a 11")
             f.write("(setv b (// 20 1))")
 
-        with pytest.raises(LexException):
+        with pytest.raises(PrematureEndOfInput):
             reload(mod)
 
         mod = sys.modules.get(TESTFN)

tests/test_bin.py

@@ -149,7 +149,7 @@ def test_bin_hy_stdin_unlocatable_hytypeerror():
     # inside run_cmd.
     _, err = run_cmd("hy", """
         (import hy.errors)
-        (raise (hy.errors.HyTypeError '[] (+ "A" "Z")))""")
+        (raise (hy.errors.HyTypeError (+ "A" "Z") None '[] None))""")
     assert "AZ" in err
 
 

tests/test_lex.py

@@ -1,13 +1,15 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
+import traceback
+
+import pytest
 
 from math import isnan
 from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol,
                        HyString, HyDict, HyList, HySet, HyKeyword)
 from hy.lex import tokenize
 from hy.lex.exceptions import LexException, PrematureEndOfInput
-import pytest
 
 def peoi(): return pytest.raises(PrematureEndOfInput)
 def lexe(): return pytest.raises(LexException)
@@ -180,7 +182,23 @@ def test_lex_digit_separators():
 
 
 def test_lex_bad_attrs():
-    with lexe(): tokenize("1.foo")
+    with lexe() as execinfo:
+        tokenize("1.foo")
+
+    expected = [
+        '  File "<string>", line 1\n',
+        '    1.foo\n',
+        '    ^\n',
+        ('LexException: Cannot access attribute on anything other'
+         ' than a name (in order to get attributes of expressions,'
+         ' use `(. <expression> <attr>)` or `(.<attr> <expression>)`)\n')
+    ]
+    output = traceback.format_exception_only(execinfo.type, execinfo.value)
+
+    assert output[:-1:1] == expected[:-1:1]
+    # Python 2.7 doesn't give the full exception name, so we compensate.
+    assert output[-1].endswith(expected[-1])
+
     with lexe(): tokenize("0.foo")
     with lexe(): tokenize("1.5.foo")
     with lexe(): tokenize("1e3.foo")