Remove hy.core compilation requirement from hy package

Previously, when importing `hy` (and any of its sub-packages/modules), Hy source
compilation for `hy.core.language` was necessarily triggered.  This, in turn,
would trigger compilation of the other standard library source files.

This commit removes that chain of events and allows the `hy` package to be
imported without any Hy compilation.

Furthermore, `read` and `read_str` are now implemented in Python and the Hy
standard library files now handle their own dependencies explicitly (i.e. they
`import` and/or `require` the other standard library files upon which they
depend).

The latter changes were necessary, because the automatically triggered
compilation of `hy.core.language` (and associated standard library files) was
serving--implicitly--as a means of producing bytecode in an order that just
happened to work for any compilation occurring afterward.  This chain of
events/dependencies was extremely cryptic, brittle, and difficult to debug, and
these changes should help to remedy that.

Closes hylang/hy#1697.

hy/__init__.py

@@ -12,5 +12,5 @@ import hy.importer  # NOQA
 # we import for side-effects.
 
 
-from hy.core.language import read, read_str, mangle, unmangle  # NOQA
+from hy.lex import read, read_str, mangle, unmangle  # NOQA
 from hy.compiler import hy_eval as eval  # NOQA

hy/compiler.py

@@ -17,6 +17,8 @@ from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
                         PY35, raise_empty)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
+import hy.core
+
 import traceback
 import importlib
 import inspect
@@ -355,20 +357,22 @@ class HyASTCompiler(object):
         self.module = module
         self.module_name = module.__name__
 
-        self.can_use_stdlib = (
-            not self.module_name.startswith("hy.core")
-            or self.module_name == "hy.core.macros")
+        # Hy expects these to be present, so we prep the module for Hy
+        # compilation.
+        self.module.__dict__.setdefault('__macros__', {})
+        self.module.__dict__.setdefault('__tags__', {})
 
-        # Load stdlib macros into the module namespace.
-        load_macros(self.module)
+        self.can_use_stdlib = not self.module_name.startswith("hy.core")
 
         self._stdlib = {}
 
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
         if self.can_use_stdlib:
+            # Load stdlib macros into the module namespace.
+            load_macros(self.module)
+
             # Populate _stdlib.
-            import hy.core
             for stdlib_module in hy.core.STDLIB:
                 mod = importlib.import_module(stdlib_module)
                 for e in map(ast_str, getattr(mod, 'EXPORTS', [])):

hy/core/language.hy

@@ -11,17 +11,19 @@
 (import [fractions [Fraction :as fraction]])
 (import operator)  ; shadow not available yet
 (import sys)
-(if-python2
-  (import [StringIO [StringIO]])
-  (import [io [StringIO]]))
 (import [hy._compat [long-type]]) ; long for python2, int for python3
+(import [hy.models [HySymbol HyKeyword]])
+(import [hy.lex [tokenize mangle unmangle read read-str]])
+(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
+(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
+
+(import [hy.core.shadow [*]])
+
+(require [hy.core.bootstrap [*]])
+
 (if-python2
   (import [collections :as cabc])
   (import [collections.abc :as cabc]))
-(import [hy.models [HySymbol HyKeyword]])
-(import [hy.lex [tokenize mangle unmangle]])
-(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
-(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
@@ -415,28 +417,6 @@ Raises ValueError for (not (pos? n))."
   "Check if `n` equals 0."
   (= n 0))
 
-(defn read [&optional [from-file sys.stdin]
-                      [eof ""]]
-  "Read from input and returns a tokenized string.
-
-Can take a given input buffer to read from, and a single byte
-as EOF (defaults to an empty string)."
-  (setv buff "")
-  (while True
-    (setv inn (string (.readline from-file)))
-    (if (= inn eof)
-      (raise (EOFError "Reached end of file")))
-    (+= buff inn)
-    (try
-      (setv parsed (first (tokenize buff)))
-      (except [e [PrematureEndOfInput IndexError]])
-      (else (break))))
-  parsed)
-
-(defn read-str [input]
-  "Reads and tokenizes first line of `input`."
-  (read :from-file (StringIO input)))
-
 (defn keyword [value]
   "Create a keyword from `value`.
 

hy/core/macros.hy

@@ -8,6 +8,11 @@
 
 (import [hy.models [HyList HySymbol]])
 
+(eval-and-compile
+  (import [hy.core.language [*]]))
+
+(require [hy.core.bootstrap [*]])
+
 (defmacro as-> [head name &rest rest]
   "Beginning with `head`, expand a sequence of assignments `rest` to `name`.
 

hy/core/shadow.hy

@@ -7,6 +7,8 @@
 (import operator)
 (import [hy._compat [PY3 PY35]])
 
+(require [hy.core.bootstrap [*]])
+
 (if PY3
   (import [functools [reduce]]))
 

hy/lex/__init__.py

@@ -5,12 +5,18 @@
 from __future__ import unicode_literals
 
 import re
+import sys
 import unicodedata
 
 from hy._compat import str_type, isidentifier, UCS4
-from hy.lex.exceptions import LexException  # NOQA
+from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
 
+try:
+    from io import StringIO
+except ImportError:
+    from StringIO import StringIO
+
 
 def hy_parse(source):
     """Parse a Hy source string.
@@ -122,3 +128,28 @@ def unicode_to_ucs4iter(ustr):
             ucs4_list[i] += ucs4_list[i + 1]
             del ucs4_list[i + 1]
     return ucs4_list
+
+
+def read(from_file=sys.stdin, eof=""):
+    """Read from input and returns a tokenized string.
+
+    Can take a given input buffer to read from, and a single byte as EOF
+    (defaults to an empty string).
+    """
+    buff = ""
+    while True:
+        inn = str(from_file.readline())
+        if inn == eof:
+            raise EOFError("Reached end of file")
+        buff += inn
+        try:
+            parsed = next(iter(tokenize(buff)), None)
+        except (PrematureEndOfInput, IndexError):
+            pass
+        else:
+            break
+    return parsed
+
+
+def read_str(input):
+    return read(StringIO(str_type(input)))