From 8b6646d5c9800c7bfb62e5a26d0b808786cd2bca Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Sat, 10 Nov 2018 16:19:41 -0600 Subject: [PATCH] Remove `hy.core` compilation requirement from `hy` package Previously, when importing `hy` (and any of its sub-packages/modules), Hy source compilation for `hy.core.language` was necessarily triggered. This, in turn, would trigger compilation of the other standard library source files. This commit removes that chain of events and allows the `hy` package to be imported without any Hy compilation. Furthermore, `read` and `read_str` are now implemented in Python and the Hy standard library files now handle their own dependencies explicitly (i.e. they `import` and/or `require` the other standard library files upon which they depend). The latter changes were necessary, because the automatically triggered compilation of `hy.core.language` (and associated standard library files) was serving--implicitly--as a means of producing bytecode in an order that just happened to work for any compilation occurring afterward. This chain of events/dependencies was extremely cryptic, brittle, and difficult to debug, and these changes should help to remedy that. Closes hylang/hy#1697. --- hy/__init__.py | 2 +- hy/compiler.py | 16 ++++++++++------ hy/core/language.hy | 38 +++++++++----------------------------- hy/core/macros.hy | 5 +++++ hy/core/shadow.hy | 2 ++ hy/lex/__init__.py | 33 ++++++++++++++++++++++++++++++++- 6 files changed, 59 insertions(+), 37 deletions(-) diff --git a/hy/__init__.py b/hy/__init__.py index 30248e0..f188b64 100644 --- a/hy/__init__.py +++ b/hy/__init__.py @@ -12,5 +12,5 @@ import hy.importer # NOQA # we import for side-effects. -from hy.core.language import read, read_str, mangle, unmangle # NOQA +from hy.lex import read, read_str, mangle, unmangle # NOQA from hy.compiler import hy_eval as eval # NOQA diff --git a/hy/compiler.py b/hy/compiler.py index 2b262a1..11473d7 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -17,6 +17,8 @@ from hy._compat import (str_type, string_types, bytes_type, long_type, PY3, PY35, raise_empty) from hy.macros import require, load_macros, macroexpand, tag_macroexpand +import hy.core + import traceback import importlib import inspect @@ -355,20 +357,22 @@ class HyASTCompiler(object): self.module = module self.module_name = module.__name__ - self.can_use_stdlib = ( - not self.module_name.startswith("hy.core") - or self.module_name == "hy.core.macros") + # Hy expects these to be present, so we prep the module for Hy + # compilation. + self.module.__dict__.setdefault('__macros__', {}) + self.module.__dict__.setdefault('__tags__', {}) - # Load stdlib macros into the module namespace. - load_macros(self.module) + self.can_use_stdlib = not self.module_name.startswith("hy.core") self._stdlib = {} # Everything in core needs to be explicit (except for # the core macros, which are built with the core functions). if self.can_use_stdlib: + # Load stdlib macros into the module namespace. + load_macros(self.module) + # Populate _stdlib. - import hy.core for stdlib_module in hy.core.STDLIB: mod = importlib.import_module(stdlib_module) for e in map(ast_str, getattr(mod, 'EXPORTS', [])): diff --git a/hy/core/language.hy b/hy/core/language.hy index 981b3d3..e13cbfa 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -11,17 +11,19 @@ (import [fractions [Fraction :as fraction]]) (import operator) ; shadow not available yet (import sys) -(if-python2 - (import [StringIO [StringIO]]) - (import [io [StringIO]])) (import [hy._compat [long-type]]) ; long for python2, int for python3 +(import [hy.models [HySymbol HyKeyword]]) +(import [hy.lex [tokenize mangle unmangle read read-str]]) +(import [hy.lex.exceptions [LexException PrematureEndOfInput]]) +(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]]) + +(import [hy.core.shadow [*]]) + +(require [hy.core.bootstrap [*]]) + (if-python2 (import [collections :as cabc]) (import [collections.abc :as cabc])) -(import [hy.models [HySymbol HyKeyword]]) -(import [hy.lex [tokenize mangle unmangle]]) -(import [hy.lex.exceptions [LexException PrematureEndOfInput]]) -(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]]) (defn butlast [coll] "Return an iterator of all but the last item in `coll`." @@ -415,28 +417,6 @@ Raises ValueError for (not (pos? n))." "Check if `n` equals 0." (= n 0)) -(defn read [&optional [from-file sys.stdin] - [eof ""]] - "Read from input and returns a tokenized string. - -Can take a given input buffer to read from, and a single byte -as EOF (defaults to an empty string)." - (setv buff "") - (while True - (setv inn (string (.readline from-file))) - (if (= inn eof) - (raise (EOFError "Reached end of file"))) - (+= buff inn) - (try - (setv parsed (first (tokenize buff))) - (except [e [PrematureEndOfInput IndexError]]) - (else (break)))) - parsed) - -(defn read-str [input] - "Reads and tokenizes first line of `input`." - (read :from-file (StringIO input))) - (defn keyword [value] "Create a keyword from `value`. diff --git a/hy/core/macros.hy b/hy/core/macros.hy index 2f9154e..0e6ab5f 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -8,6 +8,11 @@ (import [hy.models [HyList HySymbol]]) +(eval-and-compile + (import [hy.core.language [*]])) + +(require [hy.core.bootstrap [*]]) + (defmacro as-> [head name &rest rest] "Beginning with `head`, expand a sequence of assignments `rest` to `name`. diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy index ecf42a5..9fae8fe 100644 --- a/hy/core/shadow.hy +++ b/hy/core/shadow.hy @@ -7,6 +7,8 @@ (import operator) (import [hy._compat [PY3 PY35]]) +(require [hy.core.bootstrap [*]]) + (if PY3 (import [functools [reduce]])) diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py index 093fa49..2103275 100644 --- a/hy/lex/__init__.py +++ b/hy/lex/__init__.py @@ -5,12 +5,18 @@ from __future__ import unicode_literals import re +import sys import unicodedata from hy._compat import str_type, isidentifier, UCS4 -from hy.lex.exceptions import LexException # NOQA +from hy.lex.exceptions import PrematureEndOfInput, LexException # NOQA from hy.models import HyExpression, HySymbol +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + def hy_parse(source): """Parse a Hy source string. @@ -122,3 +128,28 @@ def unicode_to_ucs4iter(ustr): ucs4_list[i] += ucs4_list[i + 1] del ucs4_list[i + 1] return ucs4_list + + +def read(from_file=sys.stdin, eof=""): + """Read from input and returns a tokenized string. + + Can take a given input buffer to read from, and a single byte as EOF + (defaults to an empty string). + """ + buff = "" + while True: + inn = str(from_file.readline()) + if inn == eof: + raise EOFError("Reached end of file") + buff += inn + try: + parsed = next(iter(tokenize(buff)), None) + except (PrematureEndOfInput, IndexError): + pass + else: + break + return parsed + + +def read_str(input): + return read(StringIO(str_type(input)))