Move compilation and parsing functions out of importer.py

Functions and variables relating to compilation and parsing have been moved to
`compiler.py` and `lex/__init__.py`, respectively.  Those functions are
  - `hy_parse` from `hy.importer` to `hy.lex`
  - `hy_eval`, `ast_compile`, and `calling_module` from `hy.importer` to
  `hy.compiler`

Closes hylang/hy#1695.

hy/__init__.py

@@ -13,4 +13,4 @@ import hy.importer  # NOQA
 
 
 from hy.core.language import read, read_str, mangle, unmangle  # NOQA
-from hy.importer import hy_eval as eval  # NOQA
+from hy.compiler import hy_eval as eval  # NOQA

hy/cmdline.py

@@ -18,9 +18,10 @@ import types
 import astor.code_gen
 
 import hy
-from hy.lex import LexException, PrematureEndOfInput, mangle
+from hy.lex import hy_parse, mangle
-from hy.compiler import HyTypeError, hy_compile
+from hy.lex.exceptions import LexException, PrematureEndOfInput
-from hy.importer import hy_eval, hy_parse, runhy
+from hy.compiler import HyTypeError, hy_compile, hy_eval
+from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol

hy/compiler.py

@@ -16,7 +16,6 @@ from hy.lex import mangle, unmangle
 from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
                         PY35, raise_empty)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
-import hy.importer
 
 import traceback
 import importlib
@@ -26,6 +25,7 @@ import types
 import ast
 import sys
 import copy
+import __future__
 
 from collections import defaultdict
 
@@ -37,6 +37,60 @@ else:
 Inf = float('inf')
 
 
+hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
+                        __future__.CO_FUTURE_PRINT_FUNCTION)
+
+
+def ast_compile(ast, filename, mode):
+    """Compile AST.
+
+    Parameters
+    ----------
+    ast : instance of `ast.AST`
+
+    filename : str
+        Filename used for run-time error messages
+
+    mode: str
+        `compile` mode parameter
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    return compile(ast, filename, mode, hy_ast_compile_flags)
+
+
+def calling_module(n=1):
+    """Get the module calling, if available.
+
+    As a fallback, this will import a module using the calling frame's
+    globals value of `__name__`.
+
+    Parameters
+    ----------
+    n: int, optional
+        The number of levels up the stack from this function call.
+        The default is one level up.
+
+    Returns
+    -------
+    out: types.ModuleType
+        The module at stack level `n + 1` or `None`.
+    """
+    frame_up = inspect.stack(0)[n + 1][0]
+    module = inspect.getmodule(frame_up)
+    if module is None:
+        # This works for modules like `__main__`
+        module_name = frame_up.f_globals.get('__name__', None)
+        if module_name:
+            try:
+                module = importlib.import_module(module_name)
+            except ImportError:
+                pass
+    return module
+
+
 def ast_str(x, piecewise=False):
     if piecewise:
         return ".".join(ast_str(s) if s else "" for s in x.split("."))
@@ -1554,9 +1608,7 @@ class HyASTCompiler(object):
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
 
-        hy.importer.hy_eval(new_expr + body,
+        hy_eval(new_expr + body, self.module.__dict__, self.module)
-                            self.module.__dict__,
-                            self.module)
 
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
@@ -1723,6 +1775,83 @@ class HyASTCompiler(object):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
+def hy_eval(hytree, locals=None, module=None, ast_callback=None):
+    """Evaluates a quoted expression and returns the value.
+
+    Examples
+    --------
+
+       => (eval '(print "Hello World"))
+       "Hello World"
+
+    If you want to evaluate a string, use ``read-str`` to convert it to a
+    form first:
+
+       => (eval (read-str "(+ 1 1)"))
+       2
+
+    Parameters
+    ----------
+    hytree: a Hy expression tree
+        Source code to parse.
+
+    locals: dict, optional
+        Local environment in which to evaluate the Hy tree.  Defaults to the
+        calling frame.
+
+    module: str or types.ModuleType, optional
+        Module, or name of the module, to which the Hy tree is assigned and
+        the global values are taken.
+        Defaults to the calling frame's module, if any, and '__eval__'
+        otherwise.
+
+    ast_callback: callable, optional
+        A callback that is passed the Hy compiled tree and resulting
+        expression object, in that order, after compilation but before
+        evaluation.
+
+    Returns
+    -------
+    out : Result of evaluating the Hy compiled tree.
+    """
+    if module is None:
+        module = calling_module()
+
+    if isinstance(module, string_types):
+        module = importlib.import_module(ast_str(module, piecewise=True))
+    elif not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    if locals is None:
+        frame = inspect.stack()[1][0]
+        locals = inspect.getargvalues(frame).locals
+
+    if not isinstance(locals, dict):
+        raise TypeError("Locals must be a dictionary")
+
+    _ast, expr = hy_compile(hytree, module, get_expr=True)
+
+    # Spoof the positions in the generated ast...
+    for node in ast.walk(_ast):
+        node.lineno = 1
+        node.col_offset = 1
+
+    for node in ast.walk(expr):
+        node.lineno = 1
+        node.col_offset = 1
+
+    if ast_callback:
+        ast_callback(_ast, expr)
+
+    globals = module.__dict__
+
+    # Two-step eval: eval() the body of the exec call
+    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
+
+    # Then eval the expression context and return that
+    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
+
+
 def hy_compile(tree, module, root=ast.Module, get_expr=False):
     """
     Compile a Hy tree into a Python AST tree.

hy/core/language.hy

@@ -19,9 +19,9 @@
   (import [collections :as cabc])
   (import [collections.abc :as cabc]))
 (import [hy.models [HySymbol HyKeyword]])
-(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
+(import [hy.lex [tokenize mangle unmangle]])
-(import [hy.compiler [HyASTCompiler]])
+(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
-(import [hy.importer [calling-module hy-eval :as eval]])
+(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."

hy/importer.py

@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import sys
 import os
-import ast
 import inspect
 import pkgutil
 import re
@@ -14,163 +13,15 @@ import io
 import types
 import tempfile
 import importlib
-import __future__
 
 from functools import partial
 from contextlib import contextmanager
 
 from hy.errors import HyTypeError
-from hy.compiler import hy_compile, ast_str
+from hy.compiler import hy_compile, hy_ast_compile_flags
-from hy.lex import tokenize, LexException
+from hy.lex import hy_parse
-from hy.models import HyExpression, HySymbol
+from hy.lex.exceptions import LexException
-from hy._compat import string_types, PY3
+from hy._compat import PY3
-
-
-hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
-                        __future__.CO_FUTURE_PRINT_FUNCTION)
-
-
-def calling_module(n=1):
-    """Get the module calling, if available.
-
-    As a fallback, this will import a module using the calling frame's
-    globals value of `__name__`.
-
-    Parameters
-    ----------
-    n: int, optional
-        The number of levels up the stack from this function call.
-        The default is one level up.
-
-    Returns
-    -------
-    out: types.ModuleType
-        The module at stack level `n + 1` or `None`.
-    """
-    frame_up = inspect.stack(0)[n + 1][0]
-    module = inspect.getmodule(frame_up)
-    if module is None:
-        # This works for modules like `__main__`
-        module_name = frame_up.f_globals.get('__name__', None)
-        if module_name:
-            try:
-                module = importlib.import_module(module_name)
-            except ImportError:
-                pass
-    return module
-
-
-def ast_compile(ast, filename, mode):
-    """Compile AST.
-
-    Parameters
-    ----------
-    ast : instance of `ast.AST`
-
-    filename : str
-        Filename used for run-time error messages
-
-    mode: str
-        `compile` mode parameter
-
-    Returns
-    -------
-    out : instance of `types.CodeType`
-    """
-    return compile(ast, filename, mode, hy_ast_compile_flags)
-
-
-def hy_parse(source):
-    """Parse a Hy source string.
-
-    Parameters
-    ----------
-    source: string
-        Source code to parse.
-
-    Returns
-    -------
-    out : instance of `types.CodeType`
-    """
-    source = re.sub(r'\A#!.*', '', source)
-    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
-
-
-def hy_eval(hytree, locals=None, module=None, ast_callback=None):
-    """Evaluates a quoted expression and returns the value.
-
-    Examples
-    --------
-
-       => (eval '(print "Hello World"))
-       "Hello World"
-
-    If you want to evaluate a string, use ``read-str`` to convert it to a
-    form first:
-
-       => (eval (read-str "(+ 1 1)"))
-       2
-
-    Parameters
-    ----------
-    hytree: a Hy expression tree
-        Source code to parse.
-
-    locals: dict, optional
-        Local environment in which to evaluate the Hy tree.  Defaults to the
-        calling frame.
-
-    module: str or types.ModuleType, optional
-        Module, or name of the module, to which the Hy tree is assigned and
-        the global values are taken.
-        Defaults to the calling frame's module, if any, and '__eval__'
-        otherwise.
-
-    ast_callback: callable, optional
-        A callback that is passed the Hy compiled tree and resulting
-        expression object, in that order, after compilation but before
-        evaluation.
-
-    Returns
-    -------
-    out : Result of evaluating the Hy compiled tree.
-    """
-    if module is None:
-        module = calling_module()
-
-    if isinstance(module, string_types):
-        module = importlib.import_module(ast_str(module, piecewise=True))
-    elif not inspect.ismodule(module):
-        raise TypeError('Invalid module type: {}'.format(type(module)))
-
-    if locals is None:
-        frame = inspect.stack()[1][0]
-        locals = inspect.getargvalues(frame).locals
-
-    if not isinstance(locals, dict):
-        raise TypeError("Locals must be a dictionary")
-
-    _ast, expr = hy_compile(hytree, module, get_expr=True)
-
-    # Spoof the positions in the generated ast...
-    for node in ast.walk(_ast):
-        node.lineno = 1
-        node.col_offset = 1
-
-    for node in ast.walk(expr):
-        node.lineno = 1
-        node.col_offset = 1
-
-    if ast_callback:
-        ast_callback(_ast, expr)
-
-    globals = module.__dict__
-
-    # Two-step eval: eval() the body of the exec call
-    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
-
-    # Then eval the expression context and return that
-    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
 
 
 def cache_from_source(source_path):

hy/lex/__init__.py

@@ -4,9 +4,29 @@
 
 from __future__ import unicode_literals
 
-import re, unicodedata
+import re
+import unicodedata
+
 from hy._compat import str_type, isidentifier, UCS4
-from hy.lex.exceptions import LexException, PrematureEndOfInput  # NOQA
+from hy.lex.exceptions import LexException  # NOQA
+from hy.models import HyExpression, HySymbol
+
+
+def hy_parse(source):
+    """Parse a Hy source string.
+
+    Parameters
+    ----------
+    source: string
+        Source code to parse.
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    source = re.sub(r'\A#!.*', '', source)
+    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
+
 
 def tokenize(buf):
     """

tests/compilers/test_ast.py

@@ -7,8 +7,9 @@ from __future__ import unicode_literals
 
 from hy import HyString
 from hy.models import HyObject
-from hy.importer import hy_compile, hy_eval, hy_parse
+from hy.compiler import hy_compile, hy_eval
 from hy.errors import HyCompileError, HyTypeError
+from hy.lex import hy_parse
 from hy.lex.exceptions import LexException
 from hy._compat import PY3
 

tests/importer/test_importer.py

@@ -15,9 +15,10 @@ import pytest
 
 import hy
 from hy.errors import HyTypeError
-from hy.lex import LexException
+from hy.lex import hy_parse
+from hy.lex.exceptions import LexException
 from hy.compiler import hy_compile
-from hy.importer import hy_parse, HyLoader, cache_from_source
+from hy.importer import HyLoader, cache_from_source
 
 try:
     from importlib import reload

tests/native_tests/native_macros.hy

@@ -148,7 +148,8 @@
 (defn test-gensym-in-macros []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (setv g (gensym))
       `(do
@@ -174,7 +175,8 @@
 (defn test-with-gensym []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (with-gensyms [a]
         `(do
@@ -198,7 +200,8 @@
 (defn test-defmacro/g! []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro/g! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
@@ -227,7 +230,8 @@
   ;; defmacro! must do everything defmacro/g! can
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)

tests/test_lex.py

@@ -5,7 +5,8 @@
 from math import isnan
 from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol,
                        HyString, HyDict, HyList, HySet, HyKeyword)
-from hy.lex import LexException, PrematureEndOfInput, tokenize
+from hy.lex import tokenize
+from hy.lex.exceptions import LexException, PrematureEndOfInput
 import pytest
 
 def peoi(): return pytest.raises(PrematureEndOfInput)