Merge pull request #1699 from brandonwillard/reorganize-imports

Reorganize utility functions and imports

hy/__init__.py

@@ -12,5 +12,5 @@ import hy.importer  # NOQA
 # we import for side-effects.
 
 
-from hy.core.language import read, read_str, mangle, unmangle  # NOQA
+from hy.lex import read, read_str, mangle, unmangle  # NOQA
-from hy.importer import hy_eval as eval  # NOQA
+from hy.compiler import hy_eval as eval  # NOQA

hy/cmdline.py

@@ -18,9 +18,10 @@ import types
 import astor.code_gen
 
 import hy
-from hy.lex import LexException, PrematureEndOfInput, mangle
+from hy.lex import hy_parse, mangle
-from hy.compiler import HyTypeError, hy_compile
+from hy.lex.exceptions import LexException, PrematureEndOfInput
-from hy.importer import hy_eval, hy_parse, runhy
+from hy.compiler import HyTypeError, hy_compile, hy_eval
+from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol

hy/compiler.py

@@ -16,7 +16,8 @@ from hy.lex import mangle, unmangle
 from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
                         PY35, raise_empty)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
-import hy.importer
+
+import hy.core
 
 import traceback
 import importlib
@@ -26,6 +27,7 @@ import types
 import ast
 import sys
 import copy
+import __future__
 
 from collections import defaultdict
 
@@ -37,6 +39,60 @@ else:
 Inf = float('inf')
 
 
+hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
+                        __future__.CO_FUTURE_PRINT_FUNCTION)
+
+
+def ast_compile(ast, filename, mode):
+    """Compile AST.
+
+    Parameters
+    ----------
+    ast : instance of `ast.AST`
+
+    filename : str
+        Filename used for run-time error messages
+
+    mode: str
+        `compile` mode parameter
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    return compile(ast, filename, mode, hy_ast_compile_flags)
+
+
+def calling_module(n=1):
+    """Get the module calling, if available.
+
+    As a fallback, this will import a module using the calling frame's
+    globals value of `__name__`.
+
+    Parameters
+    ----------
+    n: int, optional
+        The number of levels up the stack from this function call.
+        The default is one level up.
+
+    Returns
+    -------
+    out: types.ModuleType
+        The module at stack level `n + 1` or `None`.
+    """
+    frame_up = inspect.stack(0)[n + 1][0]
+    module = inspect.getmodule(frame_up)
+    if module is None:
+        # This works for modules like `__main__`
+        module_name = frame_up.f_globals.get('__name__', None)
+        if module_name:
+            try:
+                module = importlib.import_module(module_name)
+            except ImportError:
+                pass
+    return module
+
+
 def ast_str(x, piecewise=False):
     if piecewise:
         return ".".join(ast_str(s) if s else "" for s in x.split("."))
@@ -301,20 +357,22 @@ class HyASTCompiler(object):
         self.module = module
         self.module_name = module.__name__
 
-        self.can_use_stdlib = (
+        # Hy expects these to be present, so we prep the module for Hy
-            not self.module_name.startswith("hy.core")
+        # compilation.
-            or self.module_name == "hy.core.macros")
+        self.module.__dict__.setdefault('__macros__', {})
+        self.module.__dict__.setdefault('__tags__', {})
 
-        # Load stdlib macros into the module namespace.
+        self.can_use_stdlib = not self.module_name.startswith("hy.core")
-        load_macros(self.module)
 
         self._stdlib = {}
 
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
         if self.can_use_stdlib:
+            # Load stdlib macros into the module namespace.
+            load_macros(self.module)
+
             # Populate _stdlib.
-            import hy.core
             for stdlib_module in hy.core.STDLIB:
                 mod = importlib.import_module(stdlib_module)
                 for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
@@ -1554,9 +1612,7 @@ class HyASTCompiler(object):
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
 
-        hy.importer.hy_eval(new_expr + body,
+        hy_eval(new_expr + body, self.module.__dict__, self.module)
-                            self.module.__dict__,
-                            self.module)
 
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
@@ -1723,6 +1779,83 @@ class HyASTCompiler(object):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
+def hy_eval(hytree, locals=None, module=None, ast_callback=None):
+    """Evaluates a quoted expression and returns the value.
+
+    Examples
+    --------
+
+       => (eval '(print "Hello World"))
+       "Hello World"
+
+    If you want to evaluate a string, use ``read-str`` to convert it to a
+    form first:
+
+       => (eval (read-str "(+ 1 1)"))
+       2
+
+    Parameters
+    ----------
+    hytree: a Hy expression tree
+        Source code to parse.
+
+    locals: dict, optional
+        Local environment in which to evaluate the Hy tree.  Defaults to the
+        calling frame.
+
+    module: str or types.ModuleType, optional
+        Module, or name of the module, to which the Hy tree is assigned and
+        the global values are taken.
+        Defaults to the calling frame's module, if any, and '__eval__'
+        otherwise.
+
+    ast_callback: callable, optional
+        A callback that is passed the Hy compiled tree and resulting
+        expression object, in that order, after compilation but before
+        evaluation.
+
+    Returns
+    -------
+    out : Result of evaluating the Hy compiled tree.
+    """
+    if module is None:
+        module = calling_module()
+
+    if isinstance(module, string_types):
+        module = importlib.import_module(ast_str(module, piecewise=True))
+    elif not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    if locals is None:
+        frame = inspect.stack()[1][0]
+        locals = inspect.getargvalues(frame).locals
+
+    if not isinstance(locals, dict):
+        raise TypeError("Locals must be a dictionary")
+
+    _ast, expr = hy_compile(hytree, module, get_expr=True)
+
+    # Spoof the positions in the generated ast...
+    for node in ast.walk(_ast):
+        node.lineno = 1
+        node.col_offset = 1
+
+    for node in ast.walk(expr):
+        node.lineno = 1
+        node.col_offset = 1
+
+    if ast_callback:
+        ast_callback(_ast, expr)
+
+    globals = module.__dict__
+
+    # Two-step eval: eval() the body of the exec call
+    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
+
+    # Then eval the expression context and return that
+    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
+
+
 def hy_compile(tree, module, root=ast.Module, get_expr=False):
     """
     Compile a Hy tree into a Python AST tree.

hy/core/language.hy

@@ -11,17 +11,19 @@
 (import [fractions [Fraction :as fraction]])
 (import operator)  ; shadow not available yet
 (import sys)
-(if-python2
-  (import [StringIO [StringIO]])
-  (import [io [StringIO]]))
 (import [hy._compat [long-type]]) ; long for python2, int for python3
+(import [hy.models [HySymbol HyKeyword]])
+(import [hy.lex [tokenize mangle unmangle read read-str]])
+(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
+(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
+
+(import [hy.core.shadow [*]])
+
+(require [hy.core.bootstrap [*]])
+
 (if-python2
   (import [collections :as cabc])
   (import [collections.abc :as cabc]))
-(import [hy.models [HySymbol HyKeyword]])
-(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
-(import [hy.compiler [HyASTCompiler]])
-(import [hy.importer [calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
@@ -415,28 +417,6 @@ Raises ValueError for (not (pos? n))."
   "Check if `n` equals 0."
   (= n 0))
 
-(defn read [&optional [from-file sys.stdin]
-                      [eof ""]]
-  "Read from input and returns a tokenized string.
-
-Can take a given input buffer to read from, and a single byte
-as EOF (defaults to an empty string)."
-  (setv buff "")
-  (while True
-    (setv inn (string (.readline from-file)))
-    (if (= inn eof)
-      (raise (EOFError "Reached end of file")))
-    (+= buff inn)
-    (try
-      (setv parsed (first (tokenize buff)))
-      (except [e [PrematureEndOfInput IndexError]])
-      (else (break))))
-  parsed)
-
-(defn read-str [input]
-  "Reads and tokenizes first line of `input`."
-  (read :from-file (StringIO input)))
-
 (defn keyword [value]
   "Create a keyword from `value`.
 

hy/core/macros.hy

@@ -8,6 +8,11 @@
 
 (import [hy.models [HyList HySymbol]])
 
+(eval-and-compile
+  (import [hy.core.language [*]]))
+
+(require [hy.core.bootstrap [*]])
+
 (defmacro as-> [head name &rest rest]
   "Beginning with `head`, expand a sequence of assignments `rest` to `name`.
 

hy/core/shadow.hy

@@ -7,6 +7,8 @@
 (import operator)
 (import [hy._compat [PY3 PY35]])
 
+(require [hy.core.bootstrap [*]])
+
 (if PY3
   (import [functools [reduce]]))
 

hy/importer.py

@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import sys
 import os
-import ast
 import inspect
 import pkgutil
 import re
@@ -14,163 +13,15 @@ import io
 import types
 import tempfile
 import importlib
-import __future__
 
 from functools import partial
 from contextlib import contextmanager
 
 from hy.errors import HyTypeError
-from hy.compiler import hy_compile, ast_str
+from hy.compiler import hy_compile, hy_ast_compile_flags
-from hy.lex import tokenize, LexException
+from hy.lex import hy_parse
-from hy.models import HyExpression, HySymbol
+from hy.lex.exceptions import LexException
-from hy._compat import string_types, PY3
+from hy._compat import PY3
-
-
-hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
-                        __future__.CO_FUTURE_PRINT_FUNCTION)
-
-
-def calling_module(n=1):
-    """Get the module calling, if available.
-
-    As a fallback, this will import a module using the calling frame's
-    globals value of `__name__`.
-
-    Parameters
-    ----------
-    n: int, optional
-        The number of levels up the stack from this function call.
-        The default is one level up.
-
-    Returns
-    -------
-    out: types.ModuleType
-        The module at stack level `n + 1` or `None`.
-    """
-    frame_up = inspect.stack(0)[n + 1][0]
-    module = inspect.getmodule(frame_up)
-    if module is None:
-        # This works for modules like `__main__`
-        module_name = frame_up.f_globals.get('__name__', None)
-        if module_name:
-            try:
-                module = importlib.import_module(module_name)
-            except ImportError:
-                pass
-    return module
-
-
-def ast_compile(ast, filename, mode):
-    """Compile AST.
-
-    Parameters
-    ----------
-    ast : instance of `ast.AST`
-
-    filename : str
-        Filename used for run-time error messages
-
-    mode: str
-        `compile` mode parameter
-
-    Returns
-    -------
-    out : instance of `types.CodeType`
-    """
-    return compile(ast, filename, mode, hy_ast_compile_flags)
-
-
-def hy_parse(source):
-    """Parse a Hy source string.
-
-    Parameters
-    ----------
-    source: string
-        Source code to parse.
-
-    Returns
-    -------
-    out : instance of `types.CodeType`
-    """
-    source = re.sub(r'\A#!.*', '', source)
-    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
-
-
-def hy_eval(hytree, locals=None, module=None, ast_callback=None):
-    """Evaluates a quoted expression and returns the value.
-
-    Examples
-    --------
-
-       => (eval '(print "Hello World"))
-       "Hello World"
-
-    If you want to evaluate a string, use ``read-str`` to convert it to a
-    form first:
-
-       => (eval (read-str "(+ 1 1)"))
-       2
-
-    Parameters
-    ----------
-    hytree: a Hy expression tree
-        Source code to parse.
-
-    locals: dict, optional
-        Local environment in which to evaluate the Hy tree.  Defaults to the
-        calling frame.
-
-    module: str or types.ModuleType, optional
-        Module, or name of the module, to which the Hy tree is assigned and
-        the global values are taken.
-        Defaults to the calling frame's module, if any, and '__eval__'
-        otherwise.
-
-    ast_callback: callable, optional
-        A callback that is passed the Hy compiled tree and resulting
-        expression object, in that order, after compilation but before
-        evaluation.
-
-    Returns
-    -------
-    out : Result of evaluating the Hy compiled tree.
-    """
-    if module is None:
-        module = calling_module()
-
-    if isinstance(module, string_types):
-        module = importlib.import_module(ast_str(module, piecewise=True))
-    elif not inspect.ismodule(module):
-        raise TypeError('Invalid module type: {}'.format(type(module)))
-
-    if locals is None:
-        frame = inspect.stack()[1][0]
-        locals = inspect.getargvalues(frame).locals
-
-    if not isinstance(locals, dict):
-        raise TypeError("Locals must be a dictionary")
-
-    _ast, expr = hy_compile(hytree, module, get_expr=True)
-
-    # Spoof the positions in the generated ast...
-    for node in ast.walk(_ast):
-        node.lineno = 1
-        node.col_offset = 1
-
-    for node in ast.walk(expr):
-        node.lineno = 1
-        node.col_offset = 1
-
-    if ast_callback:
-        ast_callback(_ast, expr)
-
-    globals = module.__dict__
-
-    # Two-step eval: eval() the body of the exec call
-    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
-
-    # Then eval the expression context and return that
-    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
 
 
 def cache_from_source(source_path):

hy/lex/__init__.py

@@ -4,9 +4,35 @@
 
 from __future__ import unicode_literals
 
-import re, unicodedata
+import re
+import sys
+import unicodedata
+
 from hy._compat import str_type, isidentifier, UCS4
-from hy.lex.exceptions import LexException, PrematureEndOfInput  # NOQA
+from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
+from hy.models import HyExpression, HySymbol
+
+try:
+    from io import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+
+def hy_parse(source):
+    """Parse a Hy source string.
+
+    Parameters
+    ----------
+    source: string
+        Source code to parse.
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    source = re.sub(r'\A#!.*', '', source)
+    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
+
 
 def tokenize(buf):
     """
@@ -102,3 +128,28 @@ def unicode_to_ucs4iter(ustr):
             ucs4_list[i] += ucs4_list[i + 1]
             del ucs4_list[i + 1]
     return ucs4_list
+
+
+def read(from_file=sys.stdin, eof=""):
+    """Read from input and returns a tokenized string.
+
+    Can take a given input buffer to read from, and a single byte as EOF
+    (defaults to an empty string).
+    """
+    buff = ""
+    while True:
+        inn = str(from_file.readline())
+        if inn == eof:
+            raise EOFError("Reached end of file")
+        buff += inn
+        try:
+            parsed = next(iter(tokenize(buff)), None)
+        except (PrematureEndOfInput, IndexError):
+            pass
+        else:
+            break
+    return parsed
+
+
+def read_str(input):
+    return read(StringIO(str_type(input)))

tests/compilers/test_ast.py

@@ -7,8 +7,9 @@ from __future__ import unicode_literals
 
 from hy import HyString
 from hy.models import HyObject
-from hy.importer import hy_compile, hy_eval, hy_parse
+from hy.compiler import hy_compile, hy_eval
 from hy.errors import HyCompileError, HyTypeError
+from hy.lex import hy_parse
 from hy.lex.exceptions import LexException
 from hy._compat import PY3
 

tests/importer/test_importer.py

@@ -15,9 +15,10 @@ import pytest
 
 import hy
 from hy.errors import HyTypeError
-from hy.lex import LexException
+from hy.lex import hy_parse
+from hy.lex.exceptions import LexException
 from hy.compiler import hy_compile
-from hy.importer import hy_parse, HyLoader, cache_from_source
+from hy.importer import HyLoader, cache_from_source
 
 try:
     from importlib import reload

tests/native_tests/native_macros.hy

@@ -148,7 +148,8 @@
 (defn test-gensym-in-macros []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (setv g (gensym))
       `(do
@@ -174,7 +175,8 @@
 (defn test-with-gensym []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (with-gensyms [a]
         `(do
@@ -198,7 +200,8 @@
 (defn test-defmacro/g! []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro/g! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
@@ -227,7 +230,8 @@
   ;; defmacro! must do everything defmacro/g! can
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)

tests/test_lex.py

@@ -5,7 +5,8 @@
 from math import isnan
 from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol,
                        HyString, HyDict, HyList, HySet, HyKeyword)
-from hy.lex import LexException, PrematureEndOfInput, tokenize
+from hy.lex import tokenize
+from hy.lex.exceptions import LexException, PrematureEndOfInput
 import pytest
 
 def peoi(): return pytest.raises(PrematureEndOfInput)