Yaltik/hy
2
0
Fork 0
Code Pull Requests Releases Activity

Merge pull request #1643 from Kodiologist/compiler-cleanup

Browse Source 
Miscellaneous compiler cleanup
This commit is contained in:
Kodi Arfer 2018-07-05 13:32:02 -07:00 committed by GitHub
commit 9253029344
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 107 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

286
hy/compiler.py
View File

@ -15,8 +15,7 @@ from hy.lex.parser import mangle, unmangle
import hy.macros import hy.macros
from hy._compat import ( from hy._compat import (
str_type, string_types, bytes_type, long_type, PY3, PY35, PY37, str_type, bytes_type, long_type, PY3, PY35, raise_empty)
raise_empty)
from hy.macros import require, macroexpand, tag_macroexpand from hy.macros import require, macroexpand, tag_macroexpand
import hy.importer import hy.importer
@ -27,7 +26,6 @@ import sys
import copy import copy
from collections import defaultdict from collections import defaultdict
from cmath import isnan
if PY3: if PY3:
import builtins import builtins
@ -36,32 +34,6 @@ else:
Inf = float('inf') Inf = float('inf')
_compile_time_ns = {}
def compile_time_ns(module_name):
ns = _compile_time_ns.get(module_name)
if ns is None:
ns = {'hy': hy, '__name__': module_name}
_compile_time_ns[module_name] = ns
return ns
_stdlib = {}
def load_stdlib():
import hy.core
if not _stdlib:
for module in hy.core.STDLIB:
mod = importlib.import_module(module)
for e in map(ast_str, mod.EXPORTS):
if getattr(mod, e) is not getattr(builtins, e, ''):
# Don't bother putting a name in _stdlib if it
# points to a builtin with the same name. This
# prevents pointless imports.
_stdlib[e] = module
def ast_str(x, piecewise=False): def ast_str(x, piecewise=False):
if piecewise: if piecewise:
@ -212,18 +184,11 @@ class Result(object):
""" """
if self.expr: if self.expr:
return self.expr return self.expr
return ast.Name(
# Spoof the position of the last statement for our generated None id=ast_str("None"),
lineno = 0 ctx=ast.Load(),
col_offset = 0 lineno=self.stmts[-1].lineno if self.stmts else 0,
if self.stmts: col_offset=self.stmts[-1].col_offset if self.stmts else 0)
lineno = self.stmts[-1].lineno
col_offset = self.stmts[-1].col_offset
return ast.Name(id=ast_str("None"),
ctx=ast.Load(),
lineno=lineno,
col_offset=col_offset)
def expr_as_stmt(self): def expr_as_stmt(self):
"""Convert the Result's expression context to a statement """Convert the Result's expression context to a statement
@ -307,36 +272,6 @@ class Result(object):
) )
def _branch(results):
"""Make a branch out of a list of Result objects
This generates a Result from the given sequence of Results, forcing each
expression context as a statement before the next result is used.
We keep the expression context of the last argument for the returned Result
"""
results = list(results)
ret = Result()
for result in results[:-1]:
ret += result
ret += result.expr_as_stmt()
for result in results[-1:]:
ret += result
return ret
def _raise_wrong_args_number(expression, error):
raise HyTypeError(expression,
error % (expression.pop(0),
len(expression)))
def _nargs(n):
return "%d argument%s" % (n, ("" if n == 1 else "s"))
def is_unpack(kind, x): def is_unpack(kind, x):
return (isinstance(x, HyExpression) return (isinstance(x, HyExpression)
and len(x) > 0 and len(x) > 0
@ -344,6 +279,9 @@ def is_unpack(kind, x):
and x[0] == "unpack-" + kind) and x[0] == "unpack-" + kind)
_stdlib = {}
class HyASTCompiler(object): class HyASTCompiler(object):
def __init__(self, module_name): def __init__(self, module_name):
@ -356,8 +294,17 @@ class HyASTCompiler(object):
or module_name == "hy.core.macros") or module_name == "hy.core.macros")
# Everything in core needs to be explicit (except for # Everything in core needs to be explicit (except for
# the core macros, which are built with the core functions). # the core macros, which are built with the core functions).
if self.can_use_stdlib: if self.can_use_stdlib and not _stdlib:
load_stdlib() # Populate _stdlib.
import hy.core
for module in hy.core.STDLIB:
mod = importlib.import_module(module)
for e in map(ast_str, mod.EXPORTS):
if getattr(mod, e) is not getattr(builtins, e, ''):
# Don't bother putting a name in _stdlib if it
# points to a builtin with the same name. This
# prevents pointless imports.
_stdlib[e] = module
def get_anon_var(self): def get_anon_var(self):
self.anon_var_count += 1 self.anon_var_count += 1
@ -401,21 +348,18 @@ class HyASTCompiler(object):
return Result() return Result()
try: try:
ret = self.compile_atom(tree) ret = self.compile_atom(tree)
if ret: self.update_imports(ret)
self.update_imports(ret) return ret
return ret
except HyCompileError: except HyCompileError:
# compile calls compile, so we're going to have multiple raise # compile calls compile, so we're going to have multiple raise
# nested; so let's re-raise this exception, let's not wrap it in # nested; so let's re-raise this exception, let's not wrap it in
# another HyCompileError! # another HyCompileError!
raise raise
except HyTypeError as e: except HyTypeError:
raise raise
except Exception as e: except Exception as e:
raise_empty(HyCompileError, e, sys.exc_info()[2]) raise_empty(HyCompileError, e, sys.exc_info()[2])
raise HyCompileError(Exception("Unknown type: `%s'" % _type))
def _compile_collect(self, exprs, with_kwargs=False, dict_display=False, def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
oldpy_unpack=False): oldpy_unpack=False):
"""Collect the expression contexts from a list of compiled expression. """Collect the expression contexts from a list of compiled expression.
@ -485,7 +429,20 @@ class HyASTCompiler(object):
return compiled_exprs, ret, keywords return compiled_exprs, ret, keywords
def _compile_branch(self, exprs): def _compile_branch(self, exprs):
return _branch(self.compile(expr) for expr in exprs) """Make a branch out of an iterable of Result objects
This generates a Result from the given sequence of Results, forcing each
expression context as a statement before the next result is used.
We keep the expression context of the last argument for the returned Result
"""
ret = Result()
for x in map(self.compile, exprs[:-1]):
ret += x
ret += x.expr_as_stmt()
if exprs:
ret += self.compile(exprs[-1])
return ret
def _storeize(self, expr, name, func=None): def _storeize(self, expr, name, func=None):
"""Return a new `name` object with an ast.Store() context""" """Return a new `name` object with an ast.Store() context"""
@ -535,68 +492,60 @@ class HyASTCompiler(object):
We need to distinguish them as want to concatenate them instead of We need to distinguish them as want to concatenate them instead of
just nesting them. just nesting them.
""" """
if level == 0:
if isinstance(form, HyExpression):
if form and form[0] in ("unquote", "unquote-splice"):
if len(form) != 2:
raise HyTypeError(form,
("`%s' needs 1 argument, got %s" %
form[0], len(form) - 1))
return set(), form[1], (form[0] == "unquote-splice")
if isinstance(form, HyExpression): op = None
if form and form[0] == "quasiquote": if isinstance(form, HyExpression) and form and (
level += 1 isinstance(form[0], HySymbol)):
if form and form[0] in ("unquote", "unquote-splice"): op = unmangle(ast_str(form[0]))
level -= 1 if level == 0 and op in ("unquote", "unquote-splice"):
if len(form) != 2:
raise HyTypeError(form,
("`%s' needs 1 argument, got %s" %
op, len(form) - 1))
return set(), form[1], op == "unquote-splice"
elif op == "quasiquote":
level += 1
elif op in ("unquote", "unquote-splice"):
level -= 1
name = form.__class__.__name__ name = form.__class__.__name__
imports = set([name]) imports = set([name])
body = [form]
if isinstance(form, HySequence): if isinstance(form, HySequence):
if not form: contents = []
contents = HyList() for x in form:
else: f_imps, f_contents, splice = self._render_quoted_form(x, level)
imports.update(f_imps)
if splice:
contents.append(HyExpression([
HySymbol("list"),
HyExpression([HySymbol("or"), f_contents, HyList()])]))
else:
contents.append(HyList([f_contents]))
if form:
# If there are arguments, they can be spliced # If there are arguments, they can be spliced
# so we build a sum... # so we build a sum...
contents = HyExpression([HySymbol("+"), HyList()]) body = [HyExpression([HySymbol("+"), HyList()] + contents)]
else:
for x in form: body = [HyList()]
f_imports, f_contents, splice = self._render_quoted_form(x,
level)
imports.update(f_imports)
if splice:
to_add = HyExpression([
HySymbol("list"),
HyExpression([HySymbol("or"), f_contents, HyList()])])
else:
to_add = HyList([f_contents])
contents.append(to_add)
return imports, HyExpression([HySymbol(name),
contents]).replace(form), False
elif isinstance(form, HySymbol): elif isinstance(form, HySymbol):
return imports, HyExpression([HySymbol(name), body = [HyString(form)]
HyString(form)]).replace(form), False
elif isinstance(form, HyKeyword): elif isinstance(form, HyKeyword):
return imports, form, False body = [HyString(form.name)]
elif isinstance(form, HyString): elif isinstance(form, HyString) and form.brackets is not None:
x = [HySymbol(name), form] body.extend([HyKeyword("brackets"), form.brackets])
if form.brackets is not None:
x.extend([HyKeyword("brackets"), form.brackets])
return imports, HyExpression(x).replace(form), False
return imports, HyExpression([HySymbol(name), ret = HyExpression([HySymbol(name)] + body).replace(form)
form]).replace(form), False return imports, ret, False
@special(["quote", "quasiquote"], [FORM]) @special(["quote", "quasiquote"], [FORM])
def compile_quote(self, expr, root, arg): def compile_quote(self, expr, root, arg):
level = Inf if root == "quote" else 0 # Only quasiquotes can unquote level = Inf if root == "quote" else 0 # Only quasiquotes can unquote
imports, stmts, splice = self._render_quoted_form(arg, level) imports, stmts, _ = self._render_quoted_form(arg, level)
ret = self.compile(stmts) ret = self.compile(stmts)
ret.add_imports("hy", imports) ret.add_imports("hy", imports)
return ret return ret
@ -626,6 +575,7 @@ class HyASTCompiler(object):
@special("raise", [maybe(FORM), maybe(sym(":from") + FORM)]) @special("raise", [maybe(FORM), maybe(sym(":from") + FORM)])
def compile_raise_expression(self, expr, root, exc, cause): def compile_raise_expression(self, expr, root, exc, cause):
ret = Result() ret = Result()
if exc is not None: if exc is not None:
exc = self.compile(exc) exc = self.compile(exc)
ret += exc ret += exc
@ -638,12 +588,10 @@ class HyASTCompiler(object):
ret += cause ret += cause
cause = cause.force_expr cause = cause.force_expr
ret += asty.Raise( return ret + asty.Raise(
expr, type=ret.expr, exc=exc, expr, type=ret.expr, exc=exc,
inst=None, tback=None, cause=cause) inst=None, tback=None, cause=cause)
return ret
@special("try", @special("try",
[many(notpexpr("except", "else", "finally")), [many(notpexpr("except", "else", "finally")),
many(pexpr(sym("except"), many(pexpr(sym("except"),
@ -736,8 +684,6 @@ class HyASTCompiler(object):
# or # or
# [] # []
# [variable [list of exceptions]]
# let's pop variable and use it as name
name = None name = None
if len(exceptions) == 2: if len(exceptions) == 2:
name = exceptions[0] name = exceptions[0]
@ -748,21 +694,20 @@ class HyASTCompiler(object):
if isinstance(exceptions_list, HyList): if isinstance(exceptions_list, HyList):
if len(exceptions_list): if len(exceptions_list):
# [FooBar BarFoo] → catch Foobar and BarFoo exceptions # [FooBar BarFoo] → catch Foobar and BarFoo exceptions
elts, _type, _ = self._compile_collect(exceptions_list) elts, types, _ = self._compile_collect(exceptions_list)
_type += asty.Tuple(exceptions_list, elts=elts, ctx=ast.Load()) types += asty.Tuple(exceptions_list, elts=elts, ctx=ast.Load())
else: else:
# [] → all exceptions caught # [] → all exceptions caught
_type = Result() types = Result()
else: else:
_type = self.compile(exceptions_list) types = self.compile(exceptions_list)
body = self._compile_branch(body) body = self._compile_branch(body)
body += asty.Assign(expr, targets=[var], value=body.force_expr) body += asty.Assign(expr, targets=[var], value=body.force_expr)
body += body.expr_as_stmt() body += body.expr_as_stmt()
# use _type.expr to get a literal `None` return types + asty.ExceptHandler(
return _type + asty.ExceptHandler( expr, type=types.expr, name=name,
expr, type=_type.expr, name=name,
body=body.stmts or [asty.Pass(expr)]) body=body.stmts or [asty.Pass(expr)])
@special("if*", [FORM, FORM, maybe(FORM)]) @special("if*", [FORM, FORM, maybe(FORM)])
@ -994,32 +939,6 @@ class HyASTCompiler(object):
elts, ret, _ = self._compile_collect(args) elts, ret, _ = self._compile_collect(args)
return ret + asty.Tuple(expr, elts=elts, ctx=ast.Load()) return ret + asty.Tuple(expr, elts=elts, ctx=ast.Load())
def _compile_generator_iterables(self, trailers):
"""Helper to compile the "trailing" parts of comprehensions:
generators and conditions"""
generators = trailers.pop(0)
cond = self.compile(trailers.pop(0)) if trailers else Result()
gen_it = iter(generators)
paired_gens = zip(gen_it, gen_it)
gen_res = Result()
gen = []
for target, iterable in paired_gens:
gen_res += self.compile(iterable)
gen.append(ast.comprehension(
target=self._storeize(target, self.compile(target)),
iter=gen_res.force_expr,
ifs=[],
is_async=False))
if cond.expr:
gen[-1].ifs.append(cond.expr)
return gen_res + cond, gen
_loopers = many( _loopers = many(
tag('setv', sym(":setv") + FORM + FORM) | tag('setv', sym(":setv") + FORM + FORM) |
tag('if', sym(":if") + FORM) | tag('if', sym(":if") + FORM) |
@ -1031,7 +950,6 @@ class HyASTCompiler(object):
@special(["lfor", "sfor", "gfor"], [_loopers, FORM]) @special(["lfor", "sfor", "gfor"], [_loopers, FORM])
@special(["dfor"], [_loopers, brackets(FORM, FORM)]) @special(["dfor"], [_loopers, brackets(FORM, FORM)])
def compile_comprehension(self, expr, root, parts, final): def compile_comprehension(self, expr, root, parts, final):
root = unmangle(ast_str(root))
node_class = { node_class = {
"for": asty.For, "for": asty.For,
"lfor": asty.ListComp, "lfor": asty.ListComp,
@ -1170,12 +1088,9 @@ class HyASTCompiler(object):
ops = {"not": ast.Not, ops = {"not": ast.Not,
"~": ast.Invert} "~": ast.Invert}
operand = self.compile(arg) operand = self.compile(arg)
return operand + asty.UnaryOp(
operand += asty.UnaryOp(
expr, op=ops[root](), operand=operand.force_expr) expr, op=ops[root](), operand=operand.force_expr)
return operand
_symn = some(lambda x: isinstance(x, HySymbol) and "." not in x) _symn = some(lambda x: isinstance(x, HySymbol) and "." not in x)
@special(["import", "require"], [many( @special(["import", "require"], [many(
@ -1213,7 +1128,7 @@ class HyASTCompiler(object):
else: else:
assignments = [(k, v or k) for k, v in kids] assignments = [(k, v or k) for k, v in kids]
if root == HySymbol("import"): if root == "import":
ast_module = ast_str(module, piecewise=True) ast_module = ast_str(module, piecewise=True)
module = ast_module.lstrip(".") module = ast_module.lstrip(".")
level = len(ast_module) - len(module) level = len(ast_module) - len(module)
@ -1235,7 +1150,7 @@ class HyASTCompiler(object):
for k, v in assignments] for k, v in assignments]
ret += node( ret += node(
expr, module=module or None, names=names, level=level) expr, module=module or None, names=names, level=level)
else: # root == HySymbol("require") else: # root == "require"
__import__(module) __import__(module)
require(module, self.module_name, require(module, self.module_name,
assignments=assignments, prefix=prefix) assignments=assignments, prefix=prefix)
@ -1247,8 +1162,9 @@ class HyASTCompiler(object):
ops = {"and": (ast.And, "True"), ops = {"and": (ast.And, "True"),
"or": (ast.Or, "None")} "or": (ast.Or, "None")}
opnode, default = ops[operator] opnode, default = ops[operator]
osym = expr[0]
if len(args) == 0: if len(args) == 0:
return asty.Name(operator, id=default, ctx=ast.Load()) return asty.Name(osym, id=default, ctx=ast.Load())
elif len(args) == 1: elif len(args) == 1:
return self.compile(args[0]) return self.compile(args[0])
ret = Result() ret = Result()
@ -1256,16 +1172,16 @@ class HyASTCompiler(object):
if any(value.stmts for value in values): if any(value.stmts for value in values):
# Compile it to an if...else sequence # Compile it to an if...else sequence
var = self.get_anon_var() var = self.get_anon_var()
name = asty.Name(operator, id=var, ctx=ast.Store()) name = asty.Name(osym, id=var, ctx=ast.Store())
expr_name = asty.Name(operator, id=var, ctx=ast.Load()) expr_name = asty.Name(osym, id=var, ctx=ast.Load())
temp_variables = [name, expr_name] temp_variables = [name, expr_name]
def make_assign(value, node=None): def make_assign(value, node=None):
positioned_name = asty.Name( positioned_name = asty.Name(
node or operator, id=var, ctx=ast.Store()) node or osym, id=var, ctx=ast.Store())
temp_variables.append(positioned_name) temp_variables.append(positioned_name)
return asty.Assign( return asty.Assign(
node or operator, targets=[positioned_name], value=value) node or osym, targets=[positioned_name], value=value)
current = root = [] current = root = []
for i, value in enumerate(values): for i, value in enumerate(values):
@ -1287,7 +1203,7 @@ class HyASTCompiler(object):
ret = sum(root, ret) ret = sum(root, ret)
ret += Result(expr=expr_name, temp_variables=temp_variables) ret += Result(expr=expr_name, temp_variables=temp_variables)
else: else:
ret += asty.BoolOp(operator, ret += asty.BoolOp(osym,
op=opnode(), op=opnode(),
values=[value.force_expr for value in values]) values=[value.force_expr for value in values])
return ret return ret
@ -1332,8 +1248,6 @@ class HyASTCompiler(object):
@special(["**", "//", "<<", ">>"], [times(2, Inf, FORM)]) @special(["**", "//", "<<", ">>"], [times(2, Inf, FORM)])
@special(["%", "^"], [times(2, 2, FORM)]) @special(["%", "^"], [times(2, 2, FORM)])
def compile_maths_expression(self, expr, root, args): def compile_maths_expression(self, expr, root, args):
root = unmangle(ast_str(root))
if len(args) == 0: if len(args) == 0:
# Return the identity element for this operator. # Return the identity element for this operator.
return asty.Num(expr, n=long_type( return asty.Num(expr, n=long_type(
@ -1370,7 +1284,7 @@ class HyASTCompiler(object):
@special(list(a_ops.keys()), [FORM, FORM]) @special(list(a_ops.keys()), [FORM, FORM])
def compile_augassign_expression(self, expr, root, target, value): def compile_augassign_expression(self, expr, root, target, value):
op = self.a_ops[unmangle(ast_str(root))] op = self.a_ops[root]
target = self._storeize(target, self.compile(target)) target = self._storeize(target, self.compile(target))
ret = self.compile(value) ret = self.compile(value)
return ret + asty.AugAssign( return ret + asty.AugAssign(
@ -1378,8 +1292,8 @@ class HyASTCompiler(object):
@special("setv", [many(FORM + FORM)]) @special("setv", [many(FORM + FORM)])
def compile_def_expression(self, expr, root, pairs): def compile_def_expression(self, expr, root, pairs):
if len(pairs) == 0: if not pairs:
return asty.Name(root, id='None', ctx=ast.Load()) return asty.Name(expr, id='None', ctx=ast.Load())
result = Result() result = Result()
for pair in pairs: for pair in pairs:
result += self._compile_assign(*pair) result += self._compile_assign(*pair)
@ -1608,11 +1522,17 @@ class HyASTCompiler(object):
arg, arg,
self)) self))
_namespaces = {}
@special(["eval-and-compile", "eval-when-compile"], [many(FORM)]) @special(["eval-and-compile", "eval-when-compile"], [many(FORM)])
def compile_eval_and_compile(self, expr, root, body): def compile_eval_and_compile(self, expr, root, body):
new_expr = HyExpression([HySymbol("do").replace(root)]).replace(expr) new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
if self.module_name not in self._namespaces:
# Initialize a compile-time namespace for this module.
self._namespaces[self.module_name] = {
'hy': hy, '__name__': self.module_name}
hy.importer.hy_eval(new_expr + body, hy.importer.hy_eval(new_expr + body,
compile_time_ns(self.module_name), self._namespaces[self.module_name],
self.module_name) self.module_name)
return (self._compile_branch(body) return (self._compile_branch(body)
if ast_str(root) == "eval_and_compile" if ast_str(root) == "eval_and_compile"
@ -1665,7 +1585,7 @@ class HyASTCompiler(object):
expression[0], expression[0],
e.msg.replace("<EOF>", "end of form"))) e.msg.replace("<EOF>", "end of form")))
return Result() + build_method( return Result() + build_method(
self, expression, expression[0], *parse_tree) self, expression, unmangle(sfn), *parse_tree)
if fn.startswith("."): if fn.startswith("."):
# (.split "test test") -> "test test".split() # (.split "test test") -> "test test".split()

2
hy/importer.py
View File

@ -5,7 +5,7 @@
from __future__ import absolute_import from __future__ import absolute_import
from hy.compiler import hy_compile, HyTypeError from hy.compiler import hy_compile, HyTypeError
from hy.models import HyObject, HyExpression, HySymbol from hy.models import HyExpression, HySymbol
from hy.lex import tokenize, LexException from hy.lex import tokenize, LexException
from hy.errors import HyIOError from hy.errors import HyIOError

4
hy/lex/parser.py
View File

@ -6,11 +6,11 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from functools import wraps from functools import wraps
import string, re, unicodedata import re, unicodedata
from rply import ParserGenerator from rply import ParserGenerator
from hy._compat import PY3, str_type, isidentifier, UCS4 from hy._compat import str_type, isidentifier, UCS4
from hy.models import (HyBytes, HyComplex, HyDict, HyExpression, HyFloat, from hy.models import (HyBytes, HyComplex, HyDict, HyExpression, HyFloat,
HyInteger, HyKeyword, HyList, HySet, HyString, HySymbol) HyInteger, HyKeyword, HyList, HySet, HyString, HySymbol)
from .lexer import lexer from .lexer import lexer

3
tests/test_bin.py
View File

@ -5,14 +5,13 @@
# license. See the LICENSE. # license. See the LICENSE.
import os import os
from pipes import quote
import re import re
import shlex import shlex
import subprocess import subprocess
import pytest import pytest
from hy._compat import PY3, PY35, PY36, builtins from hy._compat import builtins
from hy.importer import get_bytecode_path from hy.importer import get_bytecode_path