Yaltik/hy
2
0
Fork 0
Code Pull Requests Releases Activity

Merge pull request #1699 from brandonwillard/reorganize-imports

Browse Source 
Reorganize utility functions and imports
This commit is contained in:
Kodi Arfer 2018-11-28 14:23:34 -05:00 committed by GitHub
commit fb0220bd52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 237 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hy/__init__.py
View File

@ -12,5 +12,5 @@ import hy.importer # NOQA
# we import for side-effects.
from hy.core.language import read, read_str, mangle, unmangle # NOQA
from hy.importer import hy_eval as eval # NOQA
from hy.lex import read, read_str, mangle, unmangle # NOQA
from hy.compiler import hy_eval as eval # NOQA

7
hy/cmdline.py
View File

@ -18,9 +18,10 @@ import types
import astor.code_gen
import hy
from hy.lex import LexException, PrematureEndOfInput, mangle
from hy.compiler import HyTypeError, hy_compile
from hy.importer import hy_eval, hy_parse, runhy
from hy.lex import hy_parse, mangle
from hy.lex.exceptions import LexException, PrematureEndOfInput
from hy.compiler import HyTypeError, hy_compile, hy_eval
from hy.importer import runhy
from hy.completer import completion, Completer
from hy.macros import macro, require
from hy.models import HyExpression, HyString, HySymbol

153
hy/compiler.py
View File

@ -16,7 +16,8 @@ from hy.lex import mangle, unmangle
from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
PY35, raise_empty)
from hy.macros import require, load_macros, macroexpand, tag_macroexpand
import hy.importer
import hy.core
import traceback
import importlib
@ -26,6 +27,7 @@ import types
import ast
import sys
import copy
import __future__
from collections import defaultdict
@ -37,6 +39,60 @@ else:
Inf = float('inf')
hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
__future__.CO_FUTURE_PRINT_FUNCTION)
def ast_compile(ast, filename, mode):
"""Compile AST.
Parameters
----------
ast : instance of `ast.AST`
filename : str
Filename used for run-time error messages
mode: str
`compile` mode parameter
Returns
-------
out : instance of `types.CodeType`
"""
return compile(ast, filename, mode, hy_ast_compile_flags)
def calling_module(n=1):
"""Get the module calling, if available.
As a fallback, this will import a module using the calling frame's
globals value of `__name__`.
Parameters
----------
n: int, optional
The number of levels up the stack from this function call.
The default is one level up.
Returns
-------
out: types.ModuleType
The module at stack level `n + 1` or `None`.
"""
frame_up = inspect.stack(0)[n + 1][0]
module = inspect.getmodule(frame_up)
if module is None:
# This works for modules like `__main__`
module_name = frame_up.f_globals.get('__name__', None)
if module_name:
try:
module = importlib.import_module(module_name)
except ImportError:
pass
return module
def ast_str(x, piecewise=False):
if piecewise:
return ".".join(ast_str(s) if s else "" for s in x.split("."))
@ -301,20 +357,22 @@ class HyASTCompiler(object):
self.module = module
self.module_name = module.__name__
self.can_use_stdlib = (
not self.module_name.startswith("hy.core")
or self.module_name == "hy.core.macros")
# Hy expects these to be present, so we prep the module for Hy
# compilation.
self.module.__dict__.setdefault('__macros__', {})
self.module.__dict__.setdefault('__tags__', {})
# Load stdlib macros into the module namespace.
load_macros(self.module)
self.can_use_stdlib = not self.module_name.startswith("hy.core")
self._stdlib = {}
# Everything in core needs to be explicit (except for
# the core macros, which are built with the core functions).
if self.can_use_stdlib:
# Load stdlib macros into the module namespace.
load_macros(self.module)
# Populate _stdlib.
import hy.core
for stdlib_module in hy.core.STDLIB:
mod = importlib.import_module(stdlib_module)
for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
@ -1554,9 +1612,7 @@ class HyASTCompiler(object):
def compile_eval_and_compile(self, expr, root, body):
new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
hy.importer.hy_eval(new_expr + body,
self.module.__dict__,
self.module)
hy_eval(new_expr + body, self.module.__dict__, self.module)
return (self._compile_branch(body)
if ast_str(root) == "eval_and_compile"
@ -1723,6 +1779,83 @@ class HyASTCompiler(object):
return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
def hy_eval(hytree, locals=None, module=None, ast_callback=None):
"""Evaluates a quoted expression and returns the value.
Examples
--------
=> (eval '(print "Hello World"))
"Hello World"
If you want to evaluate a string, use ``read-str`` to convert it to a
form first:
=> (eval (read-str "(+ 1 1)"))
2
Parameters
----------
hytree: a Hy expression tree
Source code to parse.
locals: dict, optional
Local environment in which to evaluate the Hy tree. Defaults to the
calling frame.
module: str or types.ModuleType, optional
Module, or name of the module, to which the Hy tree is assigned and
the global values are taken.
Defaults to the calling frame's module, if any, and '__eval__'
otherwise.
ast_callback: callable, optional
A callback that is passed the Hy compiled tree and resulting
expression object, in that order, after compilation but before
evaluation.
Returns
-------
out : Result of evaluating the Hy compiled tree.
"""
if module is None:
module = calling_module()
if isinstance(module, string_types):
module = importlib.import_module(ast_str(module, piecewise=True))
elif not inspect.ismodule(module):
raise TypeError('Invalid module type: {}'.format(type(module)))
if locals is None:
frame = inspect.stack()[1][0]
locals = inspect.getargvalues(frame).locals
if not isinstance(locals, dict):
raise TypeError("Locals must be a dictionary")
_ast, expr = hy_compile(hytree, module, get_expr=True)
# Spoof the positions in the generated ast...
for node in ast.walk(_ast):
node.lineno = 1
node.col_offset = 1
for node in ast.walk(expr):
node.lineno = 1
node.col_offset = 1
if ast_callback:
ast_callback(_ast, expr)
globals = module.__dict__
# Two-step eval: eval() the body of the exec call
eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
# Then eval the expression context and return that
return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
def hy_compile(tree, module, root=ast.Module, get_expr=False):
"""
Compile a Hy tree into a Python AST tree.

38
hy/core/language.hy
View File

@ -11,17 +11,19 @@
(import [fractions [Fraction :as fraction]])
(import operator) ; shadow not available yet
(import sys)
(if-python2
(import [StringIO [StringIO]])
(import [io [StringIO]]))
(import [hy._compat [long-type]]) ; long for python2, int for python3
(import [hy.models [HySymbol HyKeyword]])
(import [hy.lex [tokenize mangle unmangle read read-str]])
(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
(import [hy.core.shadow [*]])
(require [hy.core.bootstrap [*]])
(if-python2
(import [collections :as cabc])
(import [collections.abc :as cabc]))
(import [hy.models [HySymbol HyKeyword]])
(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
(import [hy.compiler [HyASTCompiler]])
(import [hy.importer [calling-module hy-eval :as eval]])
(defn butlast [coll]
"Return an iterator of all but the last item in `coll`."
@ -415,28 +417,6 @@ Raises ValueError for (not (pos? n))."
"Check if `n` equals 0."
(= n 0))
(defn read [&optional [from-file sys.stdin]
[eof ""]]
"Read from input and returns a tokenized string.
Can take a given input buffer to read from, and a single byte
as EOF (defaults to an empty string)."
(setv buff "")
(while True
(setv inn (string (.readline from-file)))
(if (= inn eof)
(raise (EOFError "Reached end of file")))
(+= buff inn)
(try
(setv parsed (first (tokenize buff)))
(except [e [PrematureEndOfInput IndexError]])
(else (break))))
parsed)
(defn read-str [input]
"Reads and tokenizes first line of `input`."
(read :from-file (StringIO input)))
(defn keyword [value]
"Create a keyword from `value`.

5
hy/core/macros.hy
View File

@ -8,6 +8,11 @@
(import [hy.models [HyList HySymbol]])
(eval-and-compile
(import [hy.core.language [*]]))
(require [hy.core.bootstrap [*]])
(defmacro as-> [head name &rest rest]
"Beginning with `head`, expand a sequence of assignments `rest` to `name`.

2
hy/core/shadow.hy
View File

@ -7,6 +7,8 @@
(import operator)
(import [hy._compat [PY3 PY35]])
(require [hy.core.bootstrap [*]])
(if PY3
(import [functools [reduce]]))

157
hy/importer.py
View File

@ -6,7 +6,6 @@ from __future__ import absolute_import
import sys
import os
import ast
import inspect
import pkgutil
import re
@ -14,163 +13,15 @@ import io
import types
import tempfile
import importlib
import __future__
from functools import partial
from contextlib import contextmanager
from hy.errors import HyTypeError
from hy.compiler import hy_compile, ast_str
from hy.lex import tokenize, LexException
from hy.models import HyExpression, HySymbol
from hy._compat import string_types, PY3
hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
__future__.CO_FUTURE_PRINT_FUNCTION)
def calling_module(n=1):
"""Get the module calling, if available.
As a fallback, this will import a module using the calling frame's
globals value of `__name__`.
Parameters
----------
n: int, optional
The number of levels up the stack from this function call.
The default is one level up.
Returns
-------
out: types.ModuleType
The module at stack level `n + 1` or `None`.
"""
frame_up = inspect.stack(0)[n + 1][0]
module = inspect.getmodule(frame_up)
if module is None:
# This works for modules like `__main__`
module_name = frame_up.f_globals.get('__name__', None)
if module_name:
try:
module = importlib.import_module(module_name)
except ImportError:
pass
return module
def ast_compile(ast, filename, mode):
"""Compile AST.
Parameters
----------
ast : instance of `ast.AST`
filename : str
Filename used for run-time error messages
mode: str
`compile` mode parameter
Returns
-------
out : instance of `types.CodeType`
"""
return compile(ast, filename, mode, hy_ast_compile_flags)
def hy_parse(source):
"""Parse a Hy source string.
Parameters
----------
source: string
Source code to parse.
Returns
-------
out : instance of `types.CodeType`
"""
source = re.sub(r'\A#!.*', '', source)
return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
def hy_eval(hytree, locals=None, module=None, ast_callback=None):
"""Evaluates a quoted expression and returns the value.
Examples
--------
=> (eval '(print "Hello World"))
"Hello World"
If you want to evaluate a string, use ``read-str`` to convert it to a
form first:
=> (eval (read-str "(+ 1 1)"))
2
Parameters
----------
hytree: a Hy expression tree
Source code to parse.
locals: dict, optional
Local environment in which to evaluate the Hy tree. Defaults to the
calling frame.
module: str or types.ModuleType, optional
Module, or name of the module, to which the Hy tree is assigned and
the global values are taken.
Defaults to the calling frame's module, if any, and '__eval__'
otherwise.
ast_callback: callable, optional
A callback that is passed the Hy compiled tree and resulting
expression object, in that order, after compilation but before
evaluation.
Returns
-------
out : Result of evaluating the Hy compiled tree.
"""
if module is None:
module = calling_module()
if isinstance(module, string_types):
module = importlib.import_module(ast_str(module, piecewise=True))
elif not inspect.ismodule(module):
raise TypeError('Invalid module type: {}'.format(type(module)))
if locals is None:
frame = inspect.stack()[1][0]
locals = inspect.getargvalues(frame).locals
if not isinstance(locals, dict):
raise TypeError("Locals must be a dictionary")
_ast, expr = hy_compile(hytree, module, get_expr=True)
# Spoof the positions in the generated ast...
for node in ast.walk(_ast):
node.lineno = 1
node.col_offset = 1
for node in ast.walk(expr):
node.lineno = 1
node.col_offset = 1
if ast_callback:
ast_callback(_ast, expr)
globals = module.__dict__
# Two-step eval: eval() the body of the exec call
eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
# Then eval the expression context and return that
return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
from hy.compiler import hy_compile, hy_ast_compile_flags
from hy.lex import hy_parse
from hy.lex.exceptions import LexException
from hy._compat import PY3
def cache_from_source(source_path):

55
hy/lex/__init__.py
View File

@ -4,9 +4,35 @@
from __future__ import unicode_literals
import re, unicodedata
import re
import sys
import unicodedata
from hy._compat import str_type, isidentifier, UCS4
from hy.lex.exceptions import LexException, PrematureEndOfInput # NOQA
from hy.lex.exceptions import PrematureEndOfInput, LexException # NOQA
from hy.models import HyExpression, HySymbol
try:
from io import StringIO
except ImportError:
from StringIO import StringIO
def hy_parse(source):
"""Parse a Hy source string.
Parameters
----------
source: string
Source code to parse.
Returns
-------
out : instance of `types.CodeType`
"""
source = re.sub(r'\A#!.*', '', source)
return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
def tokenize(buf):
"""
@ -102,3 +128,28 @@ def unicode_to_ucs4iter(ustr):
ucs4_list[i] += ucs4_list[i + 1]
del ucs4_list[i + 1]
return ucs4_list
def read(from_file=sys.stdin, eof=""):
"""Read from input and returns a tokenized string.
Can take a given input buffer to read from, and a single byte as EOF
(defaults to an empty string).
"""
buff = ""
while True:
inn = str(from_file.readline())
if inn == eof:
raise EOFError("Reached end of file")
buff += inn
try:
parsed = next(iter(tokenize(buff)), None)
except (PrematureEndOfInput, IndexError):
pass
else:
break
return parsed
def read_str(input):
return read(StringIO(str_type(input)))

3
tests/compilers/test_ast.py
View File

@ -7,8 +7,9 @@ from __future__ import unicode_literals
from hy import HyString
from hy.models import HyObject
from hy.importer import hy_compile, hy_eval, hy_parse
from hy.compiler import hy_compile, hy_eval
from hy.errors import HyCompileError, HyTypeError
from hy.lex import hy_parse
from hy.lex.exceptions import LexException
from hy._compat import PY3

5
tests/importer/test_importer.py
View File

@ -15,9 +15,10 @@ import pytest
import hy
from hy.errors import HyTypeError
from hy.lex import LexException
from hy.lex import hy_parse
from hy.lex.exceptions import LexException
from hy.compiler import hy_compile
from hy.importer import hy_parse, HyLoader, cache_from_source
from hy.importer import HyLoader, cache_from_source
try:
from importlib import reload

12
tests/native_tests/native_macros.hy
View File

@ -148,7 +148,8 @@
(defn test-gensym-in-macros []
(import ast)
(import [astor.code-gen [to-source]])
(import [hy.importer [hy-parse hy-compile]])
(import [hy.compiler [hy-compile]])
(import [hy.lex [hy-parse]])
(setv macro1 "(defmacro nif [expr pos zero neg]
(setv g (gensym))
`(do
@ -174,7 +175,8 @@
(defn test-with-gensym []
(import ast)
(import [astor.code-gen [to-source]])
(import [hy.importer [hy-parse hy-compile]])
(import [hy.compiler [hy-compile]])
(import [hy.lex [hy-parse]])
(setv macro1 "(defmacro nif [expr pos zero neg]
(with-gensyms [a]
`(do
@ -198,7 +200,8 @@
(defn test-defmacro/g! []
(import ast)
(import [astor.code-gen [to-source]])
(import [hy.importer [hy-parse hy-compile]])
(import [hy.compiler [hy-compile]])
(import [hy.lex [hy-parse]])
(setv macro1 "(defmacro/g! nif [expr pos zero neg]
`(do
(setv ~g!res ~expr)
@ -227,7 +230,8 @@
;; defmacro! must do everything defmacro/g! can
(import ast)
(import [astor.code-gen [to-source]])
(import [hy.importer [hy-parse hy-compile]])
(import [hy.compiler [hy-compile]])
(import [hy.lex [hy-parse]])
(setv macro1 "(defmacro! nif [expr pos zero neg]
`(do
(setv ~g!res ~expr)

3
tests/test_lex.py
View File

@ -5,7 +5,8 @@
from math import isnan
from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol,
HyString, HyDict, HyList, HySet, HyKeyword)
from hy.lex import LexException, PrematureEndOfInput, tokenize
from hy.lex import tokenize
from hy.lex.exceptions import LexException, PrematureEndOfInput
import pytest
def peoi(): return pytest.raises(PrematureEndOfInput)