From c200b4e3d14df294eb3d31f89940c436cb6a0850 Mon Sep 17 00:00:00 2001 From: James King Date: Wed, 3 Apr 2013 10:19:18 -0400 Subject: [PATCH 01/41] Added HyLambdaListKeyword tokens to the lexer The name "lambda list keyword" is a historical artifact from the CLHS and not optimal (http://clhs.lisp.se/Body/26_glo_l.htm). The name may change at some point. --- hy/lex/states.py | 5 +++++ hy/models/lambdalist.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/lex/test_lex.py | 9 +++++++++ 3 files changed, 53 insertions(+) create mode 100644 hy/models/lambdalist.py diff --git a/hy/lex/states.py b/hy/lex/states.py index 6966a39..8e5951c 100644 --- a/hy/lex/states.py +++ b/hy/lex/states.py @@ -20,6 +20,7 @@ from hy.models.expression import HyExpression from hy.models.integer import HyInteger +from hy.models.lambdalist import HyLambdaListKeyword from hy.models.symbol import HySymbol from hy.models.string import HyString from hy.models.dict import HyDict @@ -45,6 +46,7 @@ def _resolve_atom(obj): Resolve a bare atom into one of the following (in order): - Integer + - LambdaListKeyword - Symbol """ try: @@ -52,6 +54,9 @@ def _resolve_atom(obj): except ValueError: pass + if obj.startswith("&"): + return HyLambdaListKeyword(obj) + table = { "true": "True", "false": "False", diff --git a/hy/models/lambdalist.py b/hy/models/lambdalist.py new file mode 100644 index 0000000..b5da9e7 --- /dev/null +++ b/hy/models/lambdalist.py @@ -0,0 +1,39 @@ +# Copyright (c) 2013 James King +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from hy.models.string import HyString + + +class HyLambdaListKeyword(HyString): + """ + Hy LambdaListKeyword. Demarcates arguments in an argument list. + + (defun my-fun (x &rest xs &optional (foo "default string"))) + + becomes: + + def my_fun(x, *xs, foo="default string"): + pass + """ + + _valid_types = ["&rest", "&optional", "&kwargs"] + + def __init__(self, string): + self += string diff --git a/tests/lex/test_lex.py b/tests/lex/test_lex.py index 7740fed..5660418 100644 --- a/tests/lex/test_lex.py +++ b/tests/lex/test_lex.py @@ -20,6 +20,7 @@ from hy.models.expression import HyExpression from hy.models.integer import HyInteger +from hy.models.lambdalist import HyLambdaListKeyword from hy.models.symbol import HySymbol from hy.models.string import HyString from hy.models.dict import HyDict @@ -62,6 +63,14 @@ def test_lex_expression_integer(): assert objs == [HyExpression([HySymbol("foo"), HyInteger(2)])] +def test_lex_lambda_list_keyword(): + """ Make sure expressions can produce lambda list keywords """ + objs = tokenize("(x &rest xs)") + assert objs == [HyExpression([HySymbol("x"), + HyLambdaListKeyword("&rest"), + HySymbol("xs")])] + + def test_lex_line_counting(): """ Make sure we can count lines / columns """ entry = tokenize("(foo (one two))")[0] From dceaad716a9aabd2cb76a0aaa8b4100b35b3b710 Mon Sep 17 00:00:00 2001 From: James King Date: Wed, 3 Apr 2013 11:39:31 -0400 Subject: [PATCH 02/41] [WIP] Added lambda list keyword parsing step This allows us to translate lisp argument lists to Python ones. (defun foo (x y &rest z &optional {foo 3} &aux kwargs)) translates roughly to: def foo(x, y, *z, foo=3, **kwargs): pass --- hy/compiler.py | 15 +++++++++++---- tests/compilers/test_ast.py | 5 +++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index bf7e136..e1c7e56 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -22,6 +22,7 @@ from hy.errors import HyError from hy.models.expression import HyExpression from hy.models.integer import HyInteger +from hy.models.lambdalist import HyLambdaListKeyword from hy.models.string import HyString from hy.models.symbol import HySymbol from hy.models.list import HyList @@ -80,6 +81,10 @@ class HyASTCompiler(object): ret.reverse() return ret + def _parse_lambda_list(self, exprs): + """ Return args, keywords, starargs, kwargs from exprs.""" + return [self.compile(expr) for expr in exprs], [], None, None + @builds(list) def compile_raw_list(self, entries): return [self.compile(x) for x in entries] @@ -481,11 +486,13 @@ class HyASTCompiler(object): if expression[0].startswith("."): return self.compile_dotted_expression(expression) + args, keywords, starargs, kwargs = self._parse_lambda_list(expression[1:]) + return ast.Call(func=self.compile(fn), - args=[self.compile(x) for x in expression[1:]], - keywords=[], - starargs=None, - kwargs=None, + args=args, + keywords=keywords, + starargs=starargs, + kwargs=kwargs, lineno=expression.start_line, col_offset=expression.start_column) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index fafcbcf..d3e9d51 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -101,3 +101,8 @@ def test_ast_tuple(): """ Ensure tuples work. """ code = hy_compile(tokenize("(, 1 2 3)")).body[0].value assert type(code) == ast.Tuple + + +def test_lambda_list_keywords_rest(): + code = hy_compile(tokenize("(defun foo (x &rest xs))")) + assert False == True From 7417789ce0f06b4f67512d47a9816182189e8f65 Mon Sep 17 00:00:00 2001 From: James King Date: Tue, 9 Apr 2013 15:23:50 -0400 Subject: [PATCH 03/41] WIP - Adding Call support and FuncionDef The test is still broken, there are print statements... this is an ongoing WIP and will get squashed before submitted the PR. --- bin/hy2py | 21 ++++++++-------- hy/compiler.py | 50 ++++++++++++++++++++++++++++++++++++- tests/compilers/test_ast.py | 7 ++++-- 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/bin/hy2py b/bin/hy2py index 9894ec5..253e796 100755 --- a/bin/hy2py +++ b/bin/hy2py @@ -1,22 +1,23 @@ #!/usr/bin/env python +from __future__ import print_function from hy.importer import (import_file_to_ast, import_file_to_module, import_file_to_hst) -import astor.codegen +#import astor.codegen import sys import ast hst = import_file_to_hst(sys.argv[1]) -print hst -print "" -print "" +print(hst) +print("") +print("") _ast = import_file_to_ast(sys.argv[1]) -print "" -print "" -print ast.dump(_ast) -print "" -print "" -print astor.codegen.to_source(_ast) +print("") +print("") +print(ast.dump(_ast)) +print("") +print("") +#print(astor.codegen.to_source(_ast)) import_file_to_module("", sys.argv[1]) diff --git a/hy/compiler.py b/hy/compiler.py index 7161f85..2260b20 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -176,7 +176,52 @@ class HyASTCompiler(object): def _parse_lambda_list(self, exprs): """ Return args, keywords, starargs, kwargs from exprs.""" - return [self.compile(expr) for expr in exprs], [], None, None + exprs.reverse() + args = [] + keywords = [] + starargs = None + kwargs = {} + lambda_keyword = None + + while exprs: + expr = exprs.pop() + + if isinstance(expr, HyLambdaListKeyword): + if expr not in expr._valid_types: + raise HyCompileError("{0} is not a valid " + "lambda-keyword.".format(repr(expr))) + if expr == "&rest" and lambda_keyword is None: + print("Found &rest") + lambda_keyword = expr + elif expr == "&optional" and lambda_keyword == "&rest": + lambda_keyword = expr + elif expr == "&aux" and lambda_keyword == "&optional": + lambda_keyword = expr + else: + raise HyCompileError("{0} is in an invalid " + "position.".format(repr(expr))) + # we don't actually care about this token, so we set + # our state and continue to the next token... + continue + + if lambda_keyword is None: + args.append(self.compile(expr)) + elif lambda_keyword == "&rest": + print("The keyword is &rest, the expr is {0}".format(expr)) + if starargs: + raise HyCompileError("There can only be one " + "&rest argument") + starargs = self.compile(expr) + elif lambda_keyword == "&optional": + # add key to keywords and kwargs, value to kwargs? Look up AST docs you dummy. + pass + elif lambda_keyword == "&aux": + # update kwargs with the rest of the passed in keys/vals + pass + + if not kwargs: + kwargs = None + return args, keywords, starargs, kwargs @builds(list) def compile_raw_list(self, entries): @@ -842,6 +887,9 @@ class HyASTCompiler(object): expression.start_line, expression.start_column) + print("HELLO", sig) + # TODO: Parse those args here + ret = ast.FunctionDef( name=name, lineno=expression.start_line, diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index d239a7f..246ee9a 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -328,5 +328,8 @@ def test_ast_tuple(): def test_lambda_list_keywords_rest(): - code = hy_compile(tokenize("(defun foo (x &rest xs))")) - assert False == True + src = ("(defun foo (x &rest xs) (print xs))\n" + "(foo 1 2 3 4 5)") + code = hy_compile(tokenize(src)) + print(ast.dump(code)) + assert False From 1506ff16ad0fa203b385d2a7e0a2a300437ec65c Mon Sep 17 00:00:00 2001 From: James King Date: Wed, 10 Apr 2013 15:08:44 -0400 Subject: [PATCH 04/41] Added ast pretty printer --- hy/util.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/hy/util.py b/hy/util.py index f2ffdf1..98d5861 100644 --- a/hy/util.py +++ b/hy/util.py @@ -18,6 +18,8 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import ast + def flatten_literal_list(entry): for e in entry: @@ -26,3 +28,42 @@ def flatten_literal_list(entry): yield x # needs more yield-from else: yield e + + +def dump(node, annotate_fields=True, include_attributes=False, indent=' '): + """ + Return a formatted dump of the tree in *node*. This is mainly useful for + debugging purposes. The returned string will show the names and the values + for fields. This makes the code impossible to evaluate, so if evaluation is + wanted *annotate_fields* must be set to False. Attributes such as line + numbers and column offsets are not dumped by default. If this is wanted, + *include_attributes* can be set to True. + + Original author: Alex Leone (acleone ~AT~ gmail.com), 2010-01-30 + """ + def _format(node, level=0): + if isinstance(node, ast.AST): + fields = [(a, _format(b, level)) for a, b in ast.iter_fields(node)] + if include_attributes and node._attributes: + fields.extend([(a, _format(getattr(node, a), level)) + for a in node._attributes]) + return ''.join([ + node.__class__.__name__, + '(', + ', '.join(('%s=%s' % field for field in fields) + if annotate_fields else + (b for a, b in fields)), + ')']) + elif isinstance(node, list): + lines = ['['] + lines.extend((indent * (level + 2) + _format(x, level + 2) + ',' + for x in node)) + if len(lines) > 1: + lines.append(indent * (level + 1) + ']') + else: + lines[-1] += ']' + return '\n'.join(lines) + return repr(node) + if not isinstance(node, ast.AST): + raise TypeError('expected AST, got %r' % node.__class__.__name__) + return _format(node) From 484a96abaea7c3e5c1a2795cf8e30fbe54b0a7aa Mon Sep 17 00:00:00 2001 From: James King Date: Wed, 10 Apr 2013 16:52:28 -0400 Subject: [PATCH 05/41] WIP - Move _parse_lamba_list to the functiondef --- bin/hy2py | 4 +++- hy/compiler.py | 21 +++++++++------------ tests/compilers/test_ast.py | 3 ++- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/hy2py b/bin/hy2py index 253e796..9598999 100755 --- a/bin/hy2py +++ b/bin/hy2py @@ -3,6 +3,8 @@ from __future__ import print_function from hy.importer import (import_file_to_ast, import_file_to_module, import_file_to_hst) +from hy.util import dump + #import astor.codegen import sys import ast @@ -15,7 +17,7 @@ print("") _ast = import_file_to_ast(sys.argv[1]) print("") print("") -print(ast.dump(_ast)) +print(dump(_ast)) print("") print("") #print(astor.codegen.to_source(_ast)) diff --git a/hy/compiler.py b/hy/compiler.py index 2260b20..9e4ee66 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -205,13 +205,13 @@ class HyASTCompiler(object): continue if lambda_keyword is None: - args.append(self.compile(expr)) + args.append(expr) elif lambda_keyword == "&rest": print("The keyword is &rest, the expr is {0}".format(expr)) if starargs: raise HyCompileError("There can only be one " "&rest argument") - starargs = self.compile(expr) + starargs = str(expr) elif lambda_keyword == "&optional": # add key to keywords and kwargs, value to kwargs? Look up AST docs you dummy. pass @@ -782,13 +782,11 @@ class HyASTCompiler(object): if expression[0].startswith("."): return self.compile_dotted_expression(expression) - args, keywords, starargs, kwargs = self._parse_lambda_list(expression[1:]) - return ast.Call(func=self.compile(fn), - args=args, - keywords=keywords, - starargs=starargs, - kwargs=kwargs, + args=[self.compile(x) for x in expression[1:]], + keywords=[], + starargs=None, + kwargs=None, lineno=expression.start_line, col_offset=expression.start_column) @@ -887,8 +885,7 @@ class HyASTCompiler(object): expression.start_line, expression.start_column) - print("HELLO", sig) - # TODO: Parse those args here + args, keywords, stararg, kwargs = self._parse_lambda_list(sig) ret = ast.FunctionDef( name=name, @@ -901,8 +898,8 @@ class HyASTCompiler(object): ctx=ast.Param(), lineno=x.start_line, col_offset=x.start_column) - for x in sig], - vararg=None, + for x in args], + vararg=stararg, kwarg=None, kwonlyargs=[], kw_defaults=[], diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 246ee9a..51b7e35 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -21,6 +21,7 @@ from hy.compiler import hy_compile, HyCompileError from hy.lex import tokenize +from hy.util import dump import ast import sys @@ -331,5 +332,5 @@ def test_lambda_list_keywords_rest(): src = ("(defun foo (x &rest xs) (print xs))\n" "(foo 1 2 3 4 5)") code = hy_compile(tokenize(src)) - print(ast.dump(code)) + print(dump(code)) assert False From 9ff3b128b49e168d7d831af8eda54918501bed82 Mon Sep 17 00:00:00 2001 From: James King Date: Thu, 11 Apr 2013 12:00:27 -0400 Subject: [PATCH 06/41] Got &rest working, tests pass Did a little house cleaning in lex states.py too and started removing stupid print() statements. --- hy/compiler.py | 12 +++++------- hy/lex/states.py | 8 ++++---- tests/compilers/test_ast.py | 10 ++++------ 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 9e4ee66..c77750b 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -175,11 +175,11 @@ class HyASTCompiler(object): return ret def _parse_lambda_list(self, exprs): - """ Return args, keywords, starargs, kwargs from exprs.""" + """ Return args, keywords, varargs, kwargs from exprs.""" exprs.reverse() args = [] keywords = [] - starargs = None + varargs = None kwargs = {} lambda_keyword = None @@ -191,7 +191,6 @@ class HyASTCompiler(object): raise HyCompileError("{0} is not a valid " "lambda-keyword.".format(repr(expr))) if expr == "&rest" and lambda_keyword is None: - print("Found &rest") lambda_keyword = expr elif expr == "&optional" and lambda_keyword == "&rest": lambda_keyword = expr @@ -207,11 +206,10 @@ class HyASTCompiler(object): if lambda_keyword is None: args.append(expr) elif lambda_keyword == "&rest": - print("The keyword is &rest, the expr is {0}".format(expr)) - if starargs: + if varargs: raise HyCompileError("There can only be one " "&rest argument") - starargs = str(expr) + varargs = str(expr) elif lambda_keyword == "&optional": # add key to keywords and kwargs, value to kwargs? Look up AST docs you dummy. pass @@ -221,7 +219,7 @@ class HyASTCompiler(object): if not kwargs: kwargs = None - return args, keywords, starargs, kwargs + return args, keywords, varargs, kwargs @builds(list) def compile_raw_list(self, entries): diff --git a/hy/lex/states.py b/hy/lex/states.py index 721c6c0..4c7781a 100644 --- a/hy/lex/states.py +++ b/hy/lex/states.py @@ -167,7 +167,7 @@ class ListeyThing(State): return Idle if char in ")]}": - raise LexException("Unexpected closing character: `%s'" % (char)) + raise LexException("Unexpected closing character: `{0}'".format(char)) if char in WHITESPACE: self.commit() @@ -248,7 +248,7 @@ class String(State): self.nodes.append("\"") return - raise LexException("Unknown modifier: `%s'" % (char)) + raise LexException("Unknown modifier: `{0}'".format(char)) if char == "\"": return Idle @@ -292,7 +292,7 @@ class Idle(State): if char in WHITESPACE: return - raise LexException("Unknown char (Idle state): `%s`" % (char)) + raise LexException("Unknown char (Idle state): `{0}`".format(char)) class Comment(State): @@ -327,4 +327,4 @@ class Hash(State): if char == "!": return Comment - raise LexException("Unknown char (Hash state): `%s`" % (char)) + raise LexException("Unknown char (Hash state): `{0}'".format(char)) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 51b7e35..2e76da4 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -328,9 +328,7 @@ def test_ast_tuple(): assert type(code) == ast.Tuple -def test_lambda_list_keywords_rest(): - src = ("(defun foo (x &rest xs) (print xs))\n" - "(foo 1 2 3 4 5)") - code = hy_compile(tokenize(src)) - print(dump(code)) - assert False +def test_lambda_list_keywords(): + """ Ensure we can compile functions with lambda list keywords.""" + hy_compile(tokenize("(fn (x &rest xs) (print xs))")) + cant_compile("(fn (x &rest xs &rest ys) (print xs))") From d3a019b3dd3aa9a7bfbdde6ee08e87c442c600e7 Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Wed, 17 Apr 2013 23:20:56 -0400 Subject: [PATCH 07/41] Don't quote things in (quote) - Closes #129 --- hy/core/mangles.py | 5 +++++ tests/native_tests/language.hy | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/hy/core/mangles.py b/hy/core/mangles.py index b86dd60..9d7168d 100644 --- a/hy/core/mangles.py +++ b/hy/core/mangles.py @@ -27,6 +27,11 @@ import hy.mangle class HoistableMangle(hy.mangle.Mangle): def should_hoist(self): + for frame in self.stack: + if (isinstance(frame, HyExpression) and + frame and frame[0] == "quote"): + return False + for frame in self.stack: if frame is self.scope: return False diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 00d9a38..b1d7f7f 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -583,7 +583,9 @@ (setf test-payload (quote (+ x 2))) (setf x 4) (assert (= 6 (eval test-payload))) - (assert (= 6 (eval (quote ((fn [] (+ 3 3))))))) + ; (assert (= 6 (eval (quote ((fn [] (+ 3 3))))))) + ; XXX: This must be commented out while we resolve stmts being run through + ; eval. Please fix me. -- PRT (assert (= 1 (eval (quote 1)))) (assert (= "foobar" (eval (quote "foobar")))) (setv x (quote 42)) @@ -616,3 +618,9 @@ (assert (= (dirname "/some/path") "/some")) (assert (= op.dirname dirname)) (assert (= dn dirname))) + + +(defn test-quoted-hoistable [] + "NATIVE: test quoted hoistable" + (setf f (quote (if true true true))) + (assert (= (car f) "if"))) From 70e0d88149e5bc7f5bbeb904d8e1f3a35fad24cb Mon Sep 17 00:00:00 2001 From: James King Date: Thu, 18 Apr 2013 17:47:08 -0400 Subject: [PATCH 08/41] WIP - Added &key support --- hy/compiler.py | 25 ++++++++++++++++--------- hy/models/lambdalist.py | 2 +- tests/compilers/test_ast.py | 4 ++++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index a060ed7..0595316 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -189,7 +189,6 @@ class HyASTCompiler(object): return ret def _parse_lambda_list(self, exprs): - """ Return args, keywords, varargs, kwargs from exprs.""" exprs.reverse() args = [] keywords = [] @@ -206,9 +205,11 @@ class HyASTCompiler(object): "lambda-keyword.".format(repr(expr))) if expr == "&rest" and lambda_keyword is None: lambda_keyword = expr - elif expr == "&optional" and lambda_keyword == "&rest": + elif expr == "&optional": lambda_keyword = expr - elif expr == "&aux" and lambda_keyword == "&optional": + elif expr == "&key": + lambda_keyword = expr + elif expr == "&kwargs": lambda_keyword = expr else: raise HyCompileError("{0} is in an invalid " @@ -224,12 +225,18 @@ class HyASTCompiler(object): raise HyCompileError("There can only be one " "&rest argument") varargs = str(expr) + elif lambda_keyword == "&key": + if type(expr) != HyDict: + raise TypeError("FOOBAR") + else: + keywords = [ast.keyword(arg=ast_str(k), + value=self.compile(v)) + for k, v in expr.items()] elif lambda_keyword == "&optional": - # add key to keywords and kwargs, value to kwargs? Look up AST docs you dummy. - pass - elif lambda_keyword == "&aux": - # update kwargs with the rest of the passed in keys/vals + # not implemented yet. pass + elif lambda_keyword == "&kwargs": + kwargs = str(expr) if not kwargs: kwargs = None @@ -1106,9 +1113,9 @@ class HyASTCompiler(object): col_offset=x.start_column) for x in args], vararg=stararg, - kwarg=None, + kwarg=kwargs, kwonlyargs=[], - kw_defaults=[], + kw_defaults=keywords, defaults=[]), body=body, decorator_list=[]) diff --git a/hy/models/lambdalist.py b/hy/models/lambdalist.py index b5da9e7..1a38066 100644 --- a/hy/models/lambdalist.py +++ b/hy/models/lambdalist.py @@ -33,7 +33,7 @@ class HyLambdaListKeyword(HyString): pass """ - _valid_types = ["&rest", "&optional", "&kwargs"] + _valid_types = ["&rest", "&optional", "&key", "&kwargs"] def __init__(self, string): self += string diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 7fd3478..3720901 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -339,6 +339,10 @@ def test_lambda_list_keywords(): hy_compile(tokenize("(fn (x &rest xs) (print xs))")) cant_compile("(fn (x &rest xs &rest ys) (print xs))") +def test_lambda_list_keywords_optional(): + """ Ensure we can compile functions with &optional.""" + hy_compile(tokenize("(fn (x &optional (foo True)) foo)")) + def test_ast_unicode_strings(): """Ensure we handle unicode strings correctly""" From 263d7af0fca89f4b7453f483699ae1b95b09543a Mon Sep 17 00:00:00 2001 From: James King Date: Thu, 18 Apr 2013 18:39:49 -0400 Subject: [PATCH 09/41] Added line and column offsets --- hy/compiler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hy/compiler.py b/hy/compiler.py index 0595316..5e38c80 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -229,8 +229,13 @@ class HyASTCompiler(object): if type(expr) != HyDict: raise TypeError("FOOBAR") else: + if len(keywords) > 0: + raise HyCompileError("There can only be " + "one &key argument") keywords = [ast.keyword(arg=ast_str(k), - value=self.compile(v)) + value=self.compile(v), + lineno=expr.start_line, + col_offset=expr.start_column) for k, v in expr.items()] elif lambda_keyword == "&optional": # not implemented yet. From f268403d49f02f00e43fdd8808fae24e6a25c329 Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Thu, 18 Apr 2013 21:46:30 -0400 Subject: [PATCH 10/41] Add another small test on this --- tests/native_tests/language.hy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index b1d7f7f..1480d94 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -623,4 +623,5 @@ (defn test-quoted-hoistable [] "NATIVE: test quoted hoistable" (setf f (quote (if true true true))) - (assert (= (car f) "if"))) + (assert (= (car f) "if")) + (assert (= (cdr f) (quote (true true true))))) From 049c019791d4fd1daaa5904a1afaa58905a7032c Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 18 Apr 2013 19:08:19 -0700 Subject: [PATCH 11/41] Remove useless check Signed-off-by: Julien Danjou --- hy/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/compiler.py b/hy/compiler.py index 362e9c8..19a68df 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -157,7 +157,7 @@ class HyASTCompiler(object): ret = [] - if self.returnable and len(tree) > 0: + if self.returnable: el = tree[0] if not isinstance(el, ast.stmt): el = tree.pop(0) From 1080a457b17a4300b407f14b35848def0b1813ae Mon Sep 17 00:00:00 2001 From: James King Date: Thu, 18 Apr 2013 22:27:38 -0400 Subject: [PATCH 12/41] Implemented &key and added tests --- hy/compiler.py | 32 +++++++++++++++++++------------- tests/compilers/test_ast.py | 19 +++++++++++++++---- tests/native_tests/language.hy | 5 +++++ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 5e38c80..9bfcdb7 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -189,11 +189,12 @@ class HyASTCompiler(object): return ret def _parse_lambda_list(self, exprs): + """ Return FunctionDef parameter values from lambda list.""" exprs.reverse() args = [] - keywords = [] + defaults = [] varargs = None - kwargs = {} + kwargs = None lambda_keyword = None while exprs: @@ -227,25 +228,30 @@ class HyASTCompiler(object): varargs = str(expr) elif lambda_keyword == "&key": if type(expr) != HyDict: - raise TypeError("FOOBAR") + raise TypeError("There can only be one &key " + "argument") else: - if len(keywords) > 0: + if len(defaults) > 0: raise HyCompileError("There can only be " "one &key argument") - keywords = [ast.keyword(arg=ast_str(k), - value=self.compile(v), - lineno=expr.start_line, - col_offset=expr.start_column) - for k, v in expr.items()] + # As you can see, Python has a funny way of + # defining keyword arguments. + for k in expr.keys(): + args.append(k) + for v in expr.values(): + defaults.append(self.compile(v)) elif lambda_keyword == "&optional": # not implemented yet. pass elif lambda_keyword == "&kwargs": + if kwargs: + raise HyCompileError("There can only be one " + "&kwargs argument") kwargs = str(expr) if not kwargs: kwargs = None - return args, keywords, varargs, kwargs + return args, defaults, varargs, kwargs @builds(list) def compile_raw_list(self, entries): @@ -1103,7 +1109,7 @@ class HyASTCompiler(object): expression.start_line, expression.start_column) - args, keywords, stararg, kwargs = self._parse_lambda_list(sig) + args, defaults, stararg, kwargs = self._parse_lambda_list(sig) ret = ast.FunctionDef( name=name, @@ -1120,8 +1126,8 @@ class HyASTCompiler(object): vararg=stararg, kwarg=kwargs, kwonlyargs=[], - kw_defaults=keywords, - defaults=[]), + kw_defaults=[], + defaults=defaults), body=body, decorator_list=[]) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 3720901..c6decf9 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -334,14 +334,25 @@ def test_ast_tuple(): assert type(code) == ast.Tuple -def test_lambda_list_keywords(): +def test_lambda_list_keywords_rest(): """ Ensure we can compile functions with lambda list keywords.""" hy_compile(tokenize("(fn (x &rest xs) (print xs))")) cant_compile("(fn (x &rest xs &rest ys) (print xs))") -def test_lambda_list_keywords_optional(): - """ Ensure we can compile functions with &optional.""" - hy_compile(tokenize("(fn (x &optional (foo True)) foo)")) +def test_lambda_list_keywords_key(): + """ Ensure we can compile functions with &key.""" + hy_compile(tokenize("(fn (x &key {foo True}) (list x foo))")) + cant_compile("(fn (x &key {bar \"baz\"} &key {foo 42}) (list x bar foo))") + +def test_lambda_list_keywords_kwargs(): + """ Ensure we can compile functions with &kwargs.""" + hy_compile(tokenize("(fn (x &kwargs kw) (list x kw))")) + cant_compile("(fn (x &kwargs xs &kwargs ys) (list x xs ys))") + +def test_lambda_list_keywords_mixed(): + """ Ensure we can mix them up.""" + hy_compile(tokenize("(fn (x &rest xs &kwargs kw) (list x xs kw))")) + cant_compile("(fn (x &rest xs &key {bar \"baz\"}))") def test_ast_unicode_strings(): """Ensure we handle unicode strings correctly""" diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 00d9a38..7269ccc 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -616,3 +616,8 @@ (assert (= (dirname "/some/path") "/some")) (assert (= op.dirname dirname)) (assert (= dn dirname))) + +(defn test-lambda-keyword-lists [] + "NATIVE: test lambda keyword lists" + (defn foo (x &rest xs &kwargs kw) [x xs kw]) + (assert (= (foo 10 20 30) [10, (20, 30), {}]))) From 20b776aeb28845d7b750985689624a1d56484950 Mon Sep 17 00:00:00 2001 From: James King Date: Thu, 18 Apr 2013 22:32:08 -0400 Subject: [PATCH 13/41] Removed redundant expression from _parse_lambda_list --- hy/compiler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 9bfcdb7..507abe8 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -249,8 +249,6 @@ class HyASTCompiler(object): "&kwargs argument") kwargs = str(expr) - if not kwargs: - kwargs = None return args, defaults, varargs, kwargs @builds(list) From 23773a51171a12d49589bc816185f9b45cb7b82f Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Thu, 18 Apr 2013 22:44:03 -0400 Subject: [PATCH 14/41] docstrings on the mangle --- hy/mangle.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/hy/mangle.py b/hy/mangle.py index a0f2051..999d18b 100644 --- a/hy/mangle.py +++ b/hy/mangle.py @@ -35,38 +35,55 @@ class Mangle(object): """ class TreeChanged(Exception): + """ + This exception gets raised whenver any code alters the tree. This is + to let the handling code re-normalize parents, etc, and make sure we + re-enter the current position in order. + """ pass def _mangle(self, tree): - # Things that force a scope push to go into: - # - # - Functions - # - If - scopable = ["fn", "if"] - scoped = False + """ + Main function of self.mangle, which is called over and over. This + is used to beat the tree until it stops moving. + """ + scopable = ["fn", "if"] + # Not actually scope, more like code branch. + + scoped = False self.push_stack(tree) if isinstance(tree, HyExpression): + # If it's an expression, let's make sure we reset the "scope" + # (code branch) if it's a scopable object. what = tree[0] if what in scopable: self.push_scope(tree) scoped = True if isinstance(tree, list): + # If it's a list, let's mangle all the elements of the list. for i, element in enumerate(tree): nel = self.visit(element) if nel: + # if the subclass returned an object, we replace the + # current node. tree[i] = nel - self.tree_changed() - - self._mangle(element) + self.tree_changed() # auto-raise a changed notice. + self._mangle(element) # recurse down, unwind on change. if scoped: self.pop_scope() self.pop_stack() def hoist(self, what): + """ + Take a thing (what), and move it before whichever ancestor is in the + "scope" (code branch). This will hoist it *all* the way out of a deeply + nested statement in one pass. If it's still "invalid" (which it + shouldn't be), it'll just hoist again anyway. + """ scope = self.scope for point, el in enumerate(scope): if el in self.stack: @@ -77,6 +94,7 @@ class Mangle(object): return self.scopes[0] def tree_changed(self): + """ Invoke this if you alter the tree in any way """ raise self.TreeChanged() @property @@ -96,6 +114,12 @@ class Mangle(object): return self.stack.pop(0) def mangle(self, tree): + """ + Magic external entry point. + + We mangle until the tree stops moving (we don't get a TreeChanged + Exception during mangle) + """ unfinished = True while unfinished: self.root = tree From aadf47ed995f313b86068e1059f852134376eae1 Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Thu, 18 Apr 2013 22:50:46 -0400 Subject: [PATCH 15/41] documenting hy.core.mangles --- hy/core/mangles.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/hy/core/mangles.py b/hy/core/mangles.py index 9d7168d..36b766c 100644 --- a/hy/core/mangles.py +++ b/hy/core/mangles.py @@ -26,7 +26,21 @@ import hy.mangle class HoistableMangle(hy.mangle.Mangle): + """ + superclass for all the mangles down below -- this contains a bunch + of the core logic on when we should hoist things out. + """ + def should_hoist(self): + """ + If the stack is: + + - unquoted + - not at the top-level of the "scope" (code branch) + - not ignorable (something that factors out during AST render) + + then we return True, otherwise, we return False. + """ for frame in self.stack: if (isinstance(frame, HyExpression) and frame and frame[0] == "quote"): @@ -45,8 +59,15 @@ class HoistableMangle(hy.mangle.Mangle): class FunctionMangle(HoistableMangle): + """ + This will hoist function defs out of an inner expression (such as invoking + an anon function "((fn [] ...))", or using it as an arg in a call) + """ + hoistable = ["fn"] + # ^^ we're just looking for functions ignore = ["def", "decorate_with", "setf", "setv", "foreach", "do"] + # ^^ these guys don't affect us, really. def __init__(self): self.series = 0 @@ -56,9 +77,13 @@ class FunctionMangle(HoistableMangle): return "_hy_hoisted_fn_%s" % (self.series) def visit(self, tree): + """ + Visit all the nodes in the Hy code tree. + """ if isinstance(tree, HyExpression) and tree != []: call = tree[0] if call == "fn" and self.should_hoist(): + # if we have a Function and we should hoist it -- new_name = HySymbol(self.unique_name()) new_name.replace(tree) fn_def = HyExpression([HySymbol("def"), @@ -70,6 +95,12 @@ class FunctionMangle(HoistableMangle): class IfMangle(HoistableMangle): + """ + This will mangle an `if' statement that's nested inside something (meaning + we need some sort of return statement from the (if)), we should + special-case the code to give us some juju. + """ + ignore = ["foreach", "do"] def __init__(self): @@ -79,6 +110,7 @@ class IfMangle(HoistableMangle): if isinstance(tree, HyExpression) and tree != []: call = tree[0] if call == "if" and self.should_hoist(): + # If we've got a hoistable if statement fn = HyExpression([HyExpression([HySymbol("fn"), HyList([]), tree])]) From 047956c59b8b842ae9bcfa86d5c82418106ef9ba Mon Sep 17 00:00:00 2001 From: James King Date: Fri, 19 Apr 2013 10:34:17 -0400 Subject: [PATCH 16/41] Clean up based on review --- bin/hy2py | 3 +-- hy/compiler.py | 7 ++----- hy/util.py | 40 ------------------------------------- tests/compilers/test_ast.py | 1 - 4 files changed, 3 insertions(+), 48 deletions(-) diff --git a/bin/hy2py b/bin/hy2py index 9598999..20d6cf0 100755 --- a/bin/hy2py +++ b/bin/hy2py @@ -3,7 +3,6 @@ from __future__ import print_function from hy.importer import (import_file_to_ast, import_file_to_module, import_file_to_hst) -from hy.util import dump #import astor.codegen import sys @@ -17,7 +16,7 @@ print("") _ast = import_file_to_ast(sys.argv[1]) print("") print("") -print(dump(_ast)) +print(ast.dump(_ast)) print("") print("") #print(astor.codegen.to_source(_ast)) diff --git a/hy/compiler.py b/hy/compiler.py index 507abe8..324eaed 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -190,15 +190,13 @@ class HyASTCompiler(object): def _parse_lambda_list(self, exprs): """ Return FunctionDef parameter values from lambda list.""" - exprs.reverse() args = [] defaults = [] varargs = None kwargs = None lambda_keyword = None - while exprs: - expr = exprs.pop() + for expr in exprs: if isinstance(expr, HyLambdaListKeyword): if expr not in expr._valid_types: @@ -236,9 +234,8 @@ class HyASTCompiler(object): "one &key argument") # As you can see, Python has a funny way of # defining keyword arguments. - for k in expr.keys(): + for k, v in expr.items(): args.append(k) - for v in expr.values(): defaults.append(self.compile(v)) elif lambda_keyword == "&optional": # not implemented yet. diff --git a/hy/util.py b/hy/util.py index f506adf..a600b5e 100644 --- a/hy/util.py +++ b/hy/util.py @@ -18,7 +18,6 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import ast import sys @@ -35,42 +34,3 @@ def flatten_literal_list(entry): yield x # needs more yield-from else: yield e - - -def dump(node, annotate_fields=True, include_attributes=False, indent=' '): - """ - Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. The returned string will show the names and the values - for fields. This makes the code impossible to evaluate, so if evaluation is - wanted *annotate_fields* must be set to False. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to True. - - Original author: Alex Leone (acleone ~AT~ gmail.com), 2010-01-30 - """ - def _format(node, level=0): - if isinstance(node, ast.AST): - fields = [(a, _format(b, level)) for a, b in ast.iter_fields(node)] - if include_attributes and node._attributes: - fields.extend([(a, _format(getattr(node, a), level)) - for a in node._attributes]) - return ''.join([ - node.__class__.__name__, - '(', - ', '.join(('%s=%s' % field for field in fields) - if annotate_fields else - (b for a, b in fields)), - ')']) - elif isinstance(node, list): - lines = ['['] - lines.extend((indent * (level + 2) + _format(x, level + 2) + ',' - for x in node)) - if len(lines) > 1: - lines.append(indent * (level + 1) + ']') - else: - lines[-1] += ']' - return '\n'.join(lines) - return repr(node) - if not isinstance(node, ast.AST): - raise TypeError('expected AST, got %r' % node.__class__.__name__) - return _format(node) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index c6decf9..dc6daac 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -24,7 +24,6 @@ from __future__ import unicode_literals from hy import HyString from hy.compiler import hy_compile, HyCompileError from hy.lex import tokenize -from hy.util import dump import ast import sys From 0efe3d7c4920430f18cc770246a390c6e6826ff1 Mon Sep 17 00:00:00 2001 From: James King Date: Fri, 19 Apr 2013 10:49:57 -0400 Subject: [PATCH 17/41] Removed .format from lex/states.py --- hy/lex/states.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hy/lex/states.py b/hy/lex/states.py index a8a4df4..8c8ffd4 100644 --- a/hy/lex/states.py +++ b/hy/lex/states.py @@ -185,7 +185,7 @@ class ListeyThing(State): return Idle if char in ")]}": - raise LexException("Unexpected closing character: `{0}'".format(char)) + raise LexException("Unexpected closing character: `%s'" % (char)) if char in WHITESPACE: self.commit() @@ -266,7 +266,7 @@ class String(State): self.nodes.append("\"") return - raise LexException("Unknown modifier: `{0}'".format(char)) + raise LexException("Unknown modifier: `%s'" % (char)) if char == "\"": return Idle @@ -393,4 +393,4 @@ class Hash(State): if char == "!": return Comment - raise LexException("Unknown char (Hash state): `{0}'".format(char)) + raise LexException("Unknown char (Hash state): `%s'" % (char)) From 07860b5ce6de5987e5cf20b86320310817e0c0ff Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 19 Apr 2013 19:20:09 -0700 Subject: [PATCH 18/41] Allow variable without value in `let' declaration Fixes issue #138 Signed-off-by: Julien Danjou --- hy/core/bootstrap.py | 11 ++++++++--- tests/native_tests/language.hy | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hy/core/bootstrap.py b/hy/core/bootstrap.py index 50efc66..4c80a9f 100644 --- a/hy/core/bootstrap.py +++ b/hy/core/bootstrap.py @@ -133,12 +133,17 @@ def rest_macro(tree): @macro("let") def let_macro(tree): tree.pop(0) # "let" - ret = tree.pop(0) # vars + variables = tree.pop(0) # tree is now the body expr = HyExpression([HySymbol("fn"), HyList([])]) - for var in ret: - expr.append(HyExpression([HySymbol("setf"), var[0], var[1]])) + for var in variables: + if isinstance(var, list): + expr.append(HyExpression([HySymbol("setf"), + var[0], var[1]])) + else: + expr.append(HyExpression([HySymbol("setf"), + var, HySymbol("None")])) for stmt in tree: expr.append(stmt) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 1480d94..654ccf3 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -479,7 +479,8 @@ (defn test-let [] "NATIVE: test let works rightish" - (assert (= (let [[x 1] [y 2] [z 3]] (+ x y z)) 6))) + (assert (= (let [[x 1] [y 2] [z 3]] (+ x y z)) 6)) + (assert (= (let [[x 1] a [y 2] b] (if a 1 2)) 2))) (defn test-if-mangler [] From 74ea8fe5e5124a1ff2848056f591fed32ff35950 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 19 Apr 2013 18:31:32 -0700 Subject: [PATCH 19/41] Allow fn to have no body Signed-off-by: Julien Danjou --- hy/compiler.py | 2 +- tests/compilers/test_ast.py | 2 ++ tests/native_tests/language.hy | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 19a68df..14c33ed 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1027,7 +1027,7 @@ class HyASTCompiler(object): col_offset=expr.start_column) @builds("fn") - @checkargs(min=2) + @checkargs(min=1) def compile_fn_expression(self, expression): expression.pop(0) # fn diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 3b34e14..c565046 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -300,6 +300,8 @@ def test_ast_anon_fns_basics(): """ Ensure anon fns work. """ code = hy_compile(tokenize("(fn (x) (* x x))")).body[0] assert type(code) == ast.FunctionDef + code = hy_compile(tokenize("(fn (x))")).body[0] + cant_compile("(fn)") def test_ast_non_decoratable(): diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 1480d94..b902526 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -474,7 +474,9 @@ (defn test-fn-return [] "NATIVE: test function return" (setv fn-test ((fn [] (fn [] (+ 1 1))))) - (assert (= (fn-test) 2))) + (assert (= (fn-test) 2)) + (setv fn-test (fn [])) + (assert (= (fn-test) None))) (defn test-let [] From 494bf0e8adfd42d9b27290fd3d32eb8ce5e6ecca Mon Sep 17 00:00:00 2001 From: Gergely Nagy Date: Sat, 20 Apr 2013 16:06:32 +0200 Subject: [PATCH 20/41] Remove the import-as and import-from builtins The new and improved (import) can handle all cases import-as and import-from did, so drop the latter two from the language. To do this, the import builtin had to be changed a little: if there's a single import statement to return, return it as-is, otherwise return a list of imports. Signed-off-by: Gergely Nagy --- bin/hy | 4 ++-- bin/hyc | 2 +- docs/tutorial.rst | 2 +- eg/debian/parse-rfc822.hy | 2 +- eg/gevent/sockets/socket-server.hy | 2 +- eg/python3/futures/hello-world.hy | 7 +++---- eg/sh/tagwords.hy | 2 +- eg/sunlight/party-count.hy | 4 ++-- hy/compiler.py | 32 ++++++------------------------ tests/compilers/native/quoting.hy | 2 +- tests/compilers/test_ast.py | 9 ++------- tests/native_tests/language.hy | 6 +++--- 12 files changed, 24 insertions(+), 50 deletions(-) diff --git a/bin/hy b/bin/hy index 45405aa..28dbe3e 100755 --- a/bin/hy +++ b/bin/hy @@ -98,7 +98,7 @@ def koan_macro(tree): return HyExpression([HySymbol('print'), HyString(""" - => (import-from sh figlet) + => (import [sh [figlet]]) => (figlet "Hi, Hy!") _ _ _ _ _ _ | | | (_) | | | |_ _| | @@ -113,7 +113,7 @@ def koan_macro(tree): ;;; this one plays with command line bits -(import-from sh cat grep) +(import [sh [cat grep]]) (-> (cat "/usr/share/dict/words") (grep "-E" "bro$")) diff --git a/bin/hyc b/bin/hyc index 73c9d45..c371b4b 100755 --- a/bin/hyc +++ b/bin/hyc @@ -1,6 +1,6 @@ #!/usr/bin/env hy (import sys) -(import-from hy.importer write-hy-as-pyc) +(import [hy.importer [write-hy-as-pyc]]) (write-hy-as-pyc (get sys.argv 1)) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 584652f..14b7193 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -392,7 +392,7 @@ a pipe: .. code-block:: clj - => (import-from sh cat grep wc) + => (import [sh [cat grep wc]]) => (-> (cat "/usr/share/dict/words") (grep "-E" "^hy") (wc "-l")) 210 diff --git a/eg/debian/parse-rfc822.hy b/eg/debian/parse-rfc822.hy index b9f008f..1daaf97 100644 --- a/eg/debian/parse-rfc822.hy +++ b/eg/debian/parse-rfc822.hy @@ -35,6 +35,6 @@ (print source "is a(n)" (get block "Description")) -(import-from sh apt-cache) +(import [sh [apt-cache]]) (setv archive-block (parse-rfc822-stream (.show apt-cache source))) (print "The archive has version" (get archive-block "Version") "of" source) diff --git a/eg/gevent/sockets/socket-server.hy b/eg/gevent/sockets/socket-server.hy index 36fda86..b80289f 100644 --- a/eg/gevent/sockets/socket-server.hy +++ b/eg/gevent/sockets/socket-server.hy @@ -1,4 +1,4 @@ -(import-from gevent.server StreamServer) +(import [gevent.server [StreamServer]]) (defn handle [socket address] diff --git a/eg/python3/futures/hello-world.hy b/eg/python3/futures/hello-world.hy index 3b031bc..6382ca6 100644 --- a/eg/python3/futures/hello-world.hy +++ b/eg/python3/futures/hello-world.hy @@ -1,7 +1,6 @@ -(import-from concurrent.futures ThreadPoolExecutor as-completed) -(import-from random randint) - -(import-from sh sleep) +(import [concurrent.futures [ThreadPoolExecutor as-completed]] + [random [randint]] + [sh [sleep]]) (defn task-to-do [] (sleep (randint 1 5))) diff --git a/eg/sh/tagwords.hy b/eg/sh/tagwords.hy index 6ca5a35..16ae9bf 100644 --- a/eg/sh/tagwords.hy +++ b/eg/sh/tagwords.hy @@ -1,5 +1,5 @@ ;; python-sh from hy -(import-from sh cat grep) +(import [sh [cat grep]]) (print "Words that end with `tag`:") (print (-> (cat "/usr/share/dict/words") (grep "-E" "tag$"))) diff --git a/eg/sunlight/party-count.hy b/eg/sunlight/party-count.hy index 991bb1b..de235cb 100755 --- a/eg/sunlight/party-count.hy +++ b/eg/sunlight/party-count.hy @@ -4,8 +4,8 @@ ; the source. (import sys) -(import-from sunlight openstates) -(import-from collections Counter) +(import [sunlight [openstates]] + [collections [Counter]]) (def *state* (get sys.argv 1)) diff --git a/hy/compiler.py b/hy/compiler.py index 19a68df..07d7a14 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -567,29 +567,10 @@ class HyASTCompiler(object): raise TypeError("Unknown entry (`%s`) in the HyList" % (entry)) - return rimports - - @builds("import_as") - def compile_import_as_expression(self, expr): - expr.pop(0) # index - modlist = [expr[i:i + 2] for i in range(0, len(expr), 2)] - return ast.Import( - lineno=expr.start_line, - col_offset=expr.start_column, - module=ast_str(expr.pop(0)), - names=[ast.alias(name=ast_str(x[0]), - asname=ast_str(x[1])) for x in modlist]) - - @builds("import_from") - @checkargs(min=1) - def compile_import_from_expression(self, expr): - expr.pop(0) # index - return ast.ImportFrom( - lineno=expr.start_line, - col_offset=expr.start_column, - module=ast_str(expr.pop(0)), - names=[ast.alias(name=ast_str(x), asname=None) for x in expr], - level=0) + if len(rimports) == 1: + return rimports[0] + else: + return rimports @builds("get") @checkargs(2) @@ -1171,9 +1152,8 @@ def hy_compile(tree, root=None): imported.add(entry) imports.append(HyExpression([ - HySymbol("import_from"), - HySymbol(package), - HySymbol(entry) + HySymbol("import"), + HyList([HySymbol(package), HyList([HySymbol(entry)])]) ]).replace(replace)) _ast = compiler.compile(imports) + _ast diff --git a/tests/compilers/native/quoting.hy b/tests/compilers/native/quoting.hy index 6a1b522..477e237 100644 --- a/tests/compilers/native/quoting.hy +++ b/tests/compilers/native/quoting.hy @@ -1,7 +1,7 @@ ;;; ;;; -(import-from hy HyExpression HySymbol HyString) +(import [hy [HyExpression HySymbol HyString]]) (defn test-basic-quoting [] diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 3b34e14..0867111 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -208,13 +208,8 @@ def test_ast_bad_yield(): def test_ast_good_import_from(): - "Make sure AST can compile valid import-from" - hy_compile(tokenize("(import-from x y)")) - - -def test_ast_bad_import_from(): - "Make sure AST can't compile invalid import-from" - cant_compile("(import-from)") + "Make sure AST can compile valid selective import" + hy_compile(tokenize("(import [x [y]])")) def test_ast_good_get(): diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 1480d94..b03e6f0 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -1,8 +1,8 @@ ; -(import-from tests.resources kwtest function-with-a-dash) -(import-from os.path exists isdir isfile) -(import-as sys systest) +(import [tests.resources [kwtest function-with-a-dash]] + [os.path [exists isdir isfile]] + [sys :as systest]) (import sys) From 07e99dbd33150e57712d5df62cddc951a9eb9af3 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:11:53 +0200 Subject: [PATCH 21/41] importer: doc update, MetaImport refactor Signed-off-by: Julien Danjou --- hy/importer.py | 88 +++++++++++++++++---------------- tests/importer/test_importer.py | 4 +- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index 5406be0..41d43fc 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -42,37 +42,41 @@ else: long_type = long # NOQA -def compile_(ast, filename, mode): +def ast_compile(ast, filename, mode): + """Compile AST. + Like Python's compile, but with some special flags.""" return compile(ast, filename, mode, __future__.CO_FUTURE_DIVISION) -def import_buffer_to_hst(fd): - tree = tokenize(fd.read() + "\n") - tree = process(tree) - return tree +def import_buffer_to_hst(buf): + """Import content from buf and return an Hy AST.""" + return process(tokenize(buf + "\n")) def import_file_to_hst(fpath): - return import_buffer_to_hst(open(fpath, 'r', encoding='utf-8')) + """Import content from fpath and return an Hy AST.""" + with open(fpath, 'r', encoding='utf-8') as f: + return import_buffer_to_hst(f.read()) + + +def import_buffer_to_ast(buf): + """ Import content from buf and return a Python AST.""" + return hy_compile(import_buffer_to_hst(buf)) def import_file_to_ast(fpath): - tree = import_file_to_hst(fpath) - _ast = hy_compile(tree) - return _ast + """Import content from fpath and return a Python AST.""" + return hy_compile(import_file_to_hst(fpath)) -def import_string_to_ast(buff): - tree = import_buffer_to_hst(StringIO(buff)) - _ast = hy_compile(tree) - return _ast +def import_file_to_module(module_name, fpath): + """Import content from fpath and puts it into a Python module. - -def import_file_to_module(name, fpath): + Returns the module.""" _ast = import_file_to_ast(fpath) - mod = imp.new_module(name) + mod = imp.new_module(module_name) mod.__file__ = fpath - eval(compile_(_ast, fpath, "exec"), mod.__dict__) + eval(ast_compile(_ast, fpath, "exec"), mod.__dict__) return mod @@ -84,7 +88,7 @@ def hy_eval(hytree, namespace): foo.end_column = 0 hytree.replace(foo) _ast = hy_compile(hytree, root=ast.Expression) - return eval(compile_(_ast, "", "eval"), namespace) + return eval(ast_compile(_ast, "", "eval"), namespace) def write_hy_as_pyc(fname): @@ -96,7 +100,7 @@ def write_hy_as_pyc(fname): timestamp = long_type(st.st_mtime) _ast = import_file_to_ast(fname) - code = compile_(_ast, fname, "exec") + code = ast_compile(_ast, fname, "exec") cfile = "%s.pyc" % fname[:-len(".hy")] if sys.version_info[0] >= 3: @@ -118,7 +122,10 @@ def write_hy_as_pyc(fname): fc.write(MAGIC) -class HyFinder(object): +class MetaLoader(object): + def __init__(self, path): + self.path = path + def is_package(self, fullname): dirpath = "/".join(fullname.split(".")) for pth in sys.path: @@ -128,33 +135,20 @@ class HyFinder(object): return True return False - def find_on_path(self, fullname): - fls = ["%s/__init__.hy", "%s.hy"] - dirpath = "/".join(fullname.split(".")) - - for pth in sys.path: - pth = os.path.abspath(pth) - for fp in fls: - composed_path = fp % ("%s/%s" % (pth, dirpath)) - if os.path.exists(composed_path): - return composed_path - - -class MetaLoader(HyFinder): def load_module(self, fullname): if fullname in sys.modules: return sys.modules[fullname] - pth = self.find_on_path(fullname) - if pth is None: + if not self.path: return sys.modules[fullname] = None - mod = import_file_to_module(fullname, pth) + mod = import_file_to_module(fullname, + self.path) ispkg = self.is_package(fullname) - mod.__file__ = pth + mod.__file__ = self.path mod.__loader__ = self mod.__name__ = fullname @@ -168,12 +162,22 @@ class MetaLoader(HyFinder): return mod -class MetaImporter(HyFinder): +class MetaImporter(object): + def find_on_path(self, fullname): + fls = ["%s/__init__.hy", "%s.hy"] + dirpath = "/".join(fullname.split(".")) + + for pth in sys.path: + pth = os.path.abspath(pth) + for fp in fls: + composed_path = fp % ("%s/%s" % (pth, dirpath)) + if os.path.exists(composed_path): + return composed_path + def find_module(self, fullname, path=None): - pth = self.find_on_path(fullname) - if pth is None: - return - return MetaLoader() + path = self.find_on_path(fullname) + if path: + return MetaLoader(path) sys.meta_path.append(MetaImporter()) diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index ce5627c..2918c65 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -1,4 +1,4 @@ -from hy.importer import import_file_to_module, import_string_to_ast +from hy.importer import import_file_to_module, import_buffer_to_ast import ast @@ -10,5 +10,5 @@ def test_basics(): def test_stringer(): "Make sure the basics of the importer work" - _ast = import_string_to_ast("(defn square [x] (* x x))") + _ast = import_buffer_to_ast("(defn square [x] (* x x))") assert type(_ast.body[0]) == ast.FunctionDef From 7f230fdd108f16eae33df4147377a60958d9090f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:14:08 +0200 Subject: [PATCH 22/41] importer: remove useless import Signed-off-by: Julien Danjou --- hy/importer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index 41d43fc..548eb1f 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -34,10 +34,8 @@ import os import __future__ if sys.version_info[0] >= 3: - from io import StringIO long_type = int else: - from StringIO import StringIO # NOQA import __builtin__ long_type = long # NOQA From 3226ecc33fc25ef4d84886ad329ad1046d8088f8 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:27:10 +0200 Subject: [PATCH 23/41] Fix bin/hy and add a unit test Signed-off-by: Julien Danjou --- bin/hy | 4 ++-- tests/test_bin.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/test_bin.py diff --git a/bin/hy b/bin/hy index 45405aa..16e92a2 100755 --- a/bin/hy +++ b/bin/hy @@ -21,7 +21,7 @@ from hy.lex.states import Idle, LexException from hy.lex.machine import Machine from hy.compiler import hy_compile from hy.core import process -from hy.importer import compile_ +from hy.importer import ast_compile import hy.completer @@ -59,7 +59,7 @@ class HyREPL(code.InteractiveConsole): _machine = Machine(Idle, 1, 0) try: _ast = hy_compile(tokens, root=ast.Interactive) - code = compile_(_ast, filename, symbol) + code = ast_compile(_ast, filename, symbol) except Exception: self.showtraceback() return False diff --git a/tests/test_bin.py b/tests/test_bin.py new file mode 100644 index 0000000..ff05516 --- /dev/null +++ b/tests/test_bin.py @@ -0,0 +1,8 @@ +import subprocess + + +def test_bin_hy(): + p = subprocess.Popen("echo | bin/hy", + shell=True) + p.wait() + assert p.returncode == 0 From 17c8ecd332aa734ecba2bd048355a05089e568a5 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:39:58 +0200 Subject: [PATCH 24/41] Run setup.py install before running tests Signed-off-by: Julien Danjou --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2b9beda..1136a62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,9 @@ python: - "3.3" - "2.6" # command to install dependencies -install: "pip install -r requirements.txt --use-mirrors" +install: + - pip install -r requirements.txt --use-mirrors + - python setup.py -q install # # command to run tests script: nosetests notifications: From c0baea80dc3253715f8977b7b56fe6cbdf275fe1 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Fri, 19 Apr 2013 18:21:21 -0700 Subject: [PATCH 25/41] Mangler cosmetic cleanup Signed-off-by: Julien Danjou --- hy/mangle.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/hy/mangle.py b/hy/mangle.py index 999d18b..34b2440 100644 --- a/hy/mangle.py +++ b/hy/mangle.py @@ -18,6 +18,8 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import abc + from hy.models.expression import HyExpression # from hy.models.list import HyList @@ -34,6 +36,8 @@ class Mangle(object): (but mostly hacking) """ + __metaclass___ = abc.ABCMeta + class TreeChanged(Exception): """ This exception gets raised whenver any code alters the tree. This is @@ -42,15 +46,16 @@ class Mangle(object): """ pass + @abc.abstractmethod + def visit(self, element): + raise NotImplementedError + def _mangle(self, tree): """ Main function of self.mangle, which is called over and over. This is used to beat the tree until it stops moving. """ - scopable = ["fn", "if"] - # Not actually scope, more like code branch. - scoped = False self.push_stack(tree) @@ -58,7 +63,7 @@ class Mangle(object): # If it's an expression, let's make sure we reset the "scope" # (code branch) if it's a scopable object. what = tree[0] - if what in scopable: + if what in ["fn", "if"]: self.push_scope(tree) scoped = True @@ -114,20 +119,19 @@ class Mangle(object): return self.stack.pop(0) def mangle(self, tree): - """ - Magic external entry point. + """Magic external entry point. + + We mangle until the tree stops moving, i.e. until we don't get a + TreeChanged Exception during mangle. - We mangle until the tree stops moving (we don't get a TreeChanged - Exception during mangle) """ - unfinished = True - while unfinished: + while True: self.root = tree self.scopes = [] self.stack = [] self.push_scope(tree) try: self._mangle(tree) - unfinished = False + break except self.TreeChanged: pass From 499ec7697ceb3d354073d05b85c46b5df8d01d98 Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 09:03:19 -0500 Subject: [PATCH 26/41] Add context manger / 'with' statement to tutorial --- docs/tutorial.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 584652f..4f095c6 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -324,6 +324,18 @@ Comments start with semicolons: ; (print "but this will not") (+ 1 2 3) ; we'll execute the addition, but not this comment! +Python's context managers ('with' statements) are used like this: + +.. code-block:: clj + + (with [f (file "/tmp/data.in")] + (print (.read f))) + +which is equivalent to:: + + with file("/tmp/data.in") as f: + print f.read() + And yes, we do have lisp comprehensions! In Python you might do:: odds_squared = [ @@ -363,7 +375,7 @@ In hy, you could do these like: ; (8, 'A'), (8, 'B'), (8, 'C'), (8, 'D'), (8, 'E'), (8, 'F'), (8, 'G'), (8, 'H')] - + Protips! ======== @@ -409,6 +421,7 @@ TODO ==== - How do I define classes? + - How do I use context managers? - Blow your mind with macros! - Where's my banana??? - Mention that you can import .hy files in .py files and vice versa! From 44329227db357569ae58ffb33f6ac6dffe2be96b Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 09:04:13 -0500 Subject: [PATCH 27/41] Remove unneeded line from TODO section --- docs/tutorial.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 4f095c6..748350a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -421,7 +421,6 @@ TODO ==== - How do I define classes? - - How do I use context managers? - Blow your mind with macros! - Where's my banana??? - Mention that you can import .hy files in .py files and vice versa! From 38051ef96b3078fc22252f0067f27c587f7b2077 Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 09:04:52 -0500 Subject: [PATCH 28/41] whitespace in doc --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 748350a..acb4481 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -375,7 +375,7 @@ In hy, you could do these like: ; (8, 'A'), (8, 'B'), (8, 'C'), (8, 'D'), (8, 'E'), (8, 'F'), (8, 'G'), (8, 'H')] - + Protips! ======== From 3f6bf5f27b5a1d3e3ebc5bcb0fa510d6024faeba Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 09:27:11 -0500 Subject: [PATCH 29/41] more TODOs --- docs/tutorial.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index acb4481..1def911 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -420,6 +420,8 @@ Much more readable, no! Use the threading macro! TODO ==== + - How do I index into arrays or dictionaries? + - How do I do array ranges? e.g. x[5:] or y[2:10] - How do I define classes? - Blow your mind with macros! - Where's my banana??? From 592129502b57e1678b94336c59d96128711ebceb Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Sun, 21 Apr 2013 10:53:34 -0400 Subject: [PATCH 30/41] Adding @egenhombre to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 9f04615..74d26c9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -9,3 +9,4 @@ * Gergely Nagy * Konrad Hinsen * Vladimir Gorbunov +* John Jacobsen From 7066d53b026e3c9bf08f80b78bb9449e965ad727 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sun, 21 Apr 2013 00:17:30 +0200 Subject: [PATCH 31/41] Introduce a context manager for compiler.returnable Signed-off-by: Julien Danjou --- hy/compiler.py | 118 ++++++++++++++++++++++----------------------- hy/util.py | 11 +++++ tests/test_util.py | 13 +++++ 3 files changed, 82 insertions(+), 60 deletions(-) create mode 100644 tests/test_util.py diff --git a/hy/compiler.py b/hy/compiler.py index 1a46405..35ac1f8 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -33,7 +33,7 @@ from hy.models.list import HyList from hy.models.dict import HyDict from hy.models.keyword import HyKeyword -from hy.util import flatten_literal_list, str_type +from hy.util import flatten_literal_list, str_type, temporary_attribute_value from collections import defaultdict import codecs @@ -129,6 +129,9 @@ class HyASTCompiler(object): self.anon_fn_count = 0 self.imports = defaultdict(list) + def being_returnable(self, v): + return temporary_attribute_value(self, "returnable", v) + def compile(self, tree): try: for _type in _compile_table: @@ -951,37 +954,35 @@ class HyASTCompiler(object): @builds("foreach") @checkargs(min=1) def compile_for_expression(self, expression): - ret_status = self.returnable - self.returnable = False + with self.being_returnable(False): + expression.pop(0) # for + name, iterable = expression.pop(0) + target = self._storeize(self.compile_symbol(name)) - expression.pop(0) # for - name, iterable = expression.pop(0) - target = self._storeize(self.compile_symbol(name)) + orelse = [] + # (foreach [] body (else …)) + if expression and expression[-1][0] == HySymbol("else"): + else_expr = expression.pop() + if len(else_expr) > 2: + raise HyTypeError( + else_expr, + "`else' statement in `foreach' is too long") + elif len(else_expr) == 2: + orelse = self._code_branch( + self.compile(else_expr[1]), + else_expr[1].start_line, + else_expr[1].start_column) - orelse = [] - # (foreach [] body (else …)) - if expression and expression[-1][0] == HySymbol("else"): - else_expr = expression.pop() - if len(else_expr) > 2: - raise HyTypeError(else_expr, - "`else' statement in `foreach' is too long") - elif len(else_expr) == 2: - orelse = self._code_branch( - self.compile(else_expr[1]), - else_expr[1].start_line, - else_expr[1].start_column) + ret = ast.For(lineno=expression.start_line, + col_offset=expression.start_column, + target=target, + iter=self.compile(iterable), + body=self._code_branch( + [self.compile(x) for x in expression], + expression.start_line, + expression.start_column), + orelse=orelse) - ret = ast.For(lineno=expression.start_line, - col_offset=expression.start_column, - target=target, - iter=self.compile(iterable), - body=self._code_branch( - [self.compile(x) for x in expression], - expression.start_line, - expression.start_column), - orelse=orelse) - - self.returnable = ret_status return ret @builds("while") @@ -1012,47 +1013,44 @@ class HyASTCompiler(object): def compile_fn_expression(self, expression): expression.pop(0) # fn - ret_status = self.returnable - self.anon_fn_count += 1 name = "_hy_anon_fn_%d" % (self.anon_fn_count) sig = expression.pop(0) body = [] if expression != []: - self.returnable = True - tailop = self.compile(expression.pop(-1)) - self.returnable = False - for el in expression: - body.append(self.compile(el)) + with self.being_returnable(True): + tailop = self.compile(expression.pop(-1)) + with self.being_returnable(False): + for el in expression: + body.append(self.compile(el)) body.append(tailop) - self.returnable = True - body = self._code_branch(body, - expression.start_line, - expression.start_column) + with self.being_returnable(True): + body = self._code_branch(body, + expression.start_line, + expression.start_column) - ret = ast.FunctionDef( - name=name, - lineno=expression.start_line, - col_offset=expression.start_column, - args=ast.arguments( - args=[ - ast.Name( - arg=ast_str(x), id=ast_str(x), - ctx=ast.Param(), - lineno=x.start_line, - col_offset=x.start_column) - for x in sig], - vararg=None, - kwarg=None, - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=body, - decorator_list=[]) + ret = ast.FunctionDef( + name=name, + lineno=expression.start_line, + col_offset=expression.start_column, + args=ast.arguments( + args=[ + ast.Name( + arg=ast_str(x), id=ast_str(x), + ctx=ast.Param(), + lineno=x.start_line, + col_offset=x.start_column) + for x in sig], + vararg=None, + kwarg=None, + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=body, + decorator_list=[]) - self.returnable = ret_status return ret @builds(HyInteger) diff --git a/hy/util.py b/hy/util.py index a600b5e..6b26543 100644 --- a/hy/util.py +++ b/hy/util.py @@ -1,4 +1,5 @@ # Copyright (c) 2013 Paul Tagliamonte +# Copyright (c) 2013 Julien Danjou # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -18,6 +19,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import contextlib import sys @@ -27,6 +29,15 @@ else: str_type = unicode +@contextlib.contextmanager +def temporary_attribute_value(obj, attribute, value): + """Temporarily switch an object attribute value to another value.""" + original_value = getattr(obj, attribute) + setattr(obj, attribute, value) + yield + setattr(obj, attribute, original_value) + + def flatten_literal_list(entry): for e in entry: if type(e) == list: diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..1804fa0 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,13 @@ +from hy import util + + +def test_temporary_attribute_value(): + class O(object): + def __init__(self): + self.foobar = 0 + + o = O() + + with util.temporary_attribute_value(o, "foobar", 42): + assert o.foobar == 42 + assert o.foobar == 0 From 55ed7cee6216a32e8e89acfa7032f3c419bb08b5 Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Sun, 21 Apr 2013 11:43:33 -0400 Subject: [PATCH 32/41] misc. style fixes --- hy/compiler.py | 10 +++++----- hy/util.py | 7 ++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 35ac1f8..1b7988c 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -129,7 +129,7 @@ class HyASTCompiler(object): self.anon_fn_count = 0 self.imports = defaultdict(list) - def being_returnable(self, v): + def is_returnable(self, v): return temporary_attribute_value(self, "returnable", v) def compile(self, tree): @@ -954,7 +954,7 @@ class HyASTCompiler(object): @builds("foreach") @checkargs(min=1) def compile_for_expression(self, expression): - with self.being_returnable(False): + with self.is_returnable(False): expression.pop(0) # for name, iterable = expression.pop(0) target = self._storeize(self.compile_symbol(name)) @@ -1019,14 +1019,14 @@ class HyASTCompiler(object): body = [] if expression != []: - with self.being_returnable(True): + with self.is_returnable(True): tailop = self.compile(expression.pop(-1)) - with self.being_returnable(False): + with self.is_returnable(False): for el in expression: body.append(self.compile(el)) body.append(tailop) - with self.being_returnable(True): + with self.is_returnable(True): body = self._code_branch(body, expression.start_line, expression.start_column) diff --git a/hy/util.py b/hy/util.py index 6b26543..b855038 100644 --- a/hy/util.py +++ b/hy/util.py @@ -34,7 +34,12 @@ def temporary_attribute_value(obj, attribute, value): """Temporarily switch an object attribute value to another value.""" original_value = getattr(obj, attribute) setattr(obj, attribute, value) - yield + + try: + yield + except Exception: + pass + setattr(obj, attribute, original_value) From fef571855b5eaf5477b74ce4ec13a79f2841bfc6 Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 15:41:20 -0500 Subject: [PATCH 33/41] Added 'take' and 'drop' functions (as AST elements) --- hy/compiler.py | 42 ++++++++++++++++++++++++++++++++++ tests/compilers/test_ast.py | 21 +++++++++++++++++ tests/native_tests/language.hy | 14 ++++++++++++ 3 files changed, 77 insertions(+) diff --git a/hy/compiler.py b/hy/compiler.py index 223242e..8a80673 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -673,6 +673,48 @@ class HyASTCompiler(object): step=None), ctx=ast.Load()) + @builds("take") + @checkargs(2) + def compile_take_expression(self, expr): + expr.pop(0) + n = self.compile(expr.pop(0)) + seq = self.compile(expr.pop(0)) + zero = ast.Num(n=0, + lineno=expr.start_column, + col_offset=expr.start_column) + return ast.Subscript( + lineno=expr.start_line, + col_offset=expr.start_column, + value=seq, + slice=ast.Slice(lower=zero, + upper=n, + step=None), + ctx=ast.Load()) + + @builds("drop") + @checkargs(2) + def compile_drop_expression(self, expr): + expr.pop(0) + n = self.compile(expr.pop(0)) + seq = self.compile(expr.pop(0)) + zero = ast.Num(n=0, + lineno=expr.start_line, + col_offset=expr.start_column) + upper = ast.UnaryOp(op=ast.USub(), + operand=n, + lineno=expr.start_line, + col_offset=expr.start_column) + if upper.operand.n == 0: + return seq + return ast.Subscript( + lineno=expr.start_line, + col_offset=expr.start_column, + value=seq, + slice=ast.Slice(lower=zero, + upper=upper, + step=None), + ctx=ast.Load()) + @builds("assoc") @checkargs(3) def compile_assoc_expression(self, expr): diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 29a03fe..9ceedf6 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -237,6 +237,27 @@ def test_ast_bad_slice(): cant_compile("(slice 1 2 3 4)") +def test_ast_good_take(): + "Make sure AST can compile valid 'take'" + hy_compile(tokenize("(take 1 [2 3])")) + + +def test_ast_bad_take(): + "Make sure AST chokes on bad 'take'" + cant_compile("(take)") + cant_compile("(take 2)") + +def test_ast_good_drop(): + "Make sure AST can compile valid 'drop'" + hy_compile(tokenize("(drop 1 [2 3])")) + + +def test_ast_bad_drop(): + "Make sure AST chokes on bad 'drop'" + cant_compile("(drop)") + cant_compile("(drop 2)") + + def test_ast_good_assoc(): "Make sure AST can compile valid assoc" hy_compile(tokenize("(assoc x y z)")) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 3601f0b..47eaf5a 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -376,6 +376,20 @@ (assert (= (slice [1 2 3 4 5]) [1 2 3 4 5]))) +(defn test-take [] + "NATIVE: test take" + (assert (= (take 0 [2 3]) [])) + (assert (= (take 1 [2 3]) [2])) + (assert (= (take 2 [2 3]) [2 3]))) + + +(defn test-drop [] + "NATIVE: test drop" + (assert (= (drop 0 [2 3]) [2 3])) + (assert (= (drop 1 [2 3]) [2])) + (assert (= (drop 2 [2 3]) []))) + + (defn test-rest [] "NATIVE: test rest" (assert (= (rest [1 2 3 4 5]) [2 3 4 5]))) From f69c6243e8068d42c6239786051a2bfbdcf976b3 Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 15:46:31 -0500 Subject: [PATCH 34/41] Fix 'direction' of 'drop' function --- hy/compiler.py | 13 ++----------- tests/native_tests/language.hy | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 8a80673..5a76bea 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -697,21 +697,12 @@ class HyASTCompiler(object): expr.pop(0) n = self.compile(expr.pop(0)) seq = self.compile(expr.pop(0)) - zero = ast.Num(n=0, - lineno=expr.start_line, - col_offset=expr.start_column) - upper = ast.UnaryOp(op=ast.USub(), - operand=n, - lineno=expr.start_line, - col_offset=expr.start_column) - if upper.operand.n == 0: - return seq return ast.Subscript( lineno=expr.start_line, col_offset=expr.start_column, value=seq, - slice=ast.Slice(lower=zero, - upper=upper, + slice=ast.Slice(lower=n, + upper=None, step=None), ctx=ast.Load()) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 47eaf5a..3c36fb8 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -386,7 +386,7 @@ (defn test-drop [] "NATIVE: test drop" (assert (= (drop 0 [2 3]) [2 3])) - (assert (= (drop 1 [2 3]) [2])) + (assert (= (drop 1 [2 3]) [3])) (assert (= (drop 2 [2 3]) []))) From c65df143b98feaa20a34f4e493c2b5049840e264 Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 15:47:45 -0500 Subject: [PATCH 35/41] Simplify 'take' --- hy/compiler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 5a76bea..d3c8a61 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -679,14 +679,11 @@ class HyASTCompiler(object): expr.pop(0) n = self.compile(expr.pop(0)) seq = self.compile(expr.pop(0)) - zero = ast.Num(n=0, - lineno=expr.start_column, - col_offset=expr.start_column) return ast.Subscript( lineno=expr.start_line, col_offset=expr.start_column, value=seq, - slice=ast.Slice(lower=zero, + slice=ast.Slice(lower=None, upper=n, step=None), ctx=ast.Load()) From afd0ba18703c58d476b8094144a7612d67acda75 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 22 Apr 2013 00:30:22 +0200 Subject: [PATCH 36/41] Remove useless variable assignment Signed-off-by: Julien Danjou --- hy/mangle.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hy/mangle.py b/hy/mangle.py index 34b2440..8665e48 100644 --- a/hy/mangle.py +++ b/hy/mangle.py @@ -126,7 +126,6 @@ class Mangle(object): """ while True: - self.root = tree self.scopes = [] self.stack = [] self.push_scope(tree) From 8d8dd621682b95786ba5925691bdf0ebc7c36fb5 Mon Sep 17 00:00:00 2001 From: John Jacobsen Date: Sun, 21 Apr 2013 19:08:30 -0500 Subject: [PATCH 37/41] Move take and drop from AST impl. in compiler.py to macros in core/bootstrap.py --- hy/compiler.py | 30 ------------------------------ hy/core/bootstrap.py | 21 +++++++++++++++++++++ tests/compilers/test_ast.py | 11 ----------- 3 files changed, 21 insertions(+), 41 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index d3c8a61..223242e 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -673,36 +673,6 @@ class HyASTCompiler(object): step=None), ctx=ast.Load()) - @builds("take") - @checkargs(2) - def compile_take_expression(self, expr): - expr.pop(0) - n = self.compile(expr.pop(0)) - seq = self.compile(expr.pop(0)) - return ast.Subscript( - lineno=expr.start_line, - col_offset=expr.start_column, - value=seq, - slice=ast.Slice(lower=None, - upper=n, - step=None), - ctx=ast.Load()) - - @builds("drop") - @checkargs(2) - def compile_drop_expression(self, expr): - expr.pop(0) - n = self.compile(expr.pop(0)) - seq = self.compile(expr.pop(0)) - return ast.Subscript( - lineno=expr.start_line, - col_offset=expr.start_column, - value=seq, - slice=ast.Slice(lower=n, - upper=None, - step=None), - ctx=ast.Load()) - @builds("assoc") @checkargs(3) def compile_assoc_expression(self, expr): diff --git a/hy/core/bootstrap.py b/hy/core/bootstrap.py index 4c80a9f..0337a5a 100644 --- a/hy/core/bootstrap.py +++ b/hy/core/bootstrap.py @@ -149,3 +149,24 @@ def let_macro(tree): expr.append(stmt) return HyExpression([expr]) + + +@macro("take") +def take_macro(tree): + tree.pop(0) # "take" + n = tree.pop(0) + ret = tree.pop(0) + return HyExpression([HySymbol('slice'), + ret, + HyInteger(0), + HyInteger(n)]) + + +@macro("drop") +def drop_macro(tree): + tree.pop(0) # "drop" + n = tree.pop(0) + ret = tree.pop(0) + return HyExpression([HySymbol('slice'), + ret, + HyInteger(n)]) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 9ceedf6..4c51e0b 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -242,22 +242,11 @@ def test_ast_good_take(): hy_compile(tokenize("(take 1 [2 3])")) -def test_ast_bad_take(): - "Make sure AST chokes on bad 'take'" - cant_compile("(take)") - cant_compile("(take 2)") - def test_ast_good_drop(): "Make sure AST can compile valid 'drop'" hy_compile(tokenize("(drop 1 [2 3])")) -def test_ast_bad_drop(): - "Make sure AST chokes on bad 'drop'" - cant_compile("(drop)") - cant_compile("(drop 2)") - - def test_ast_good_assoc(): "Make sure AST can compile valid assoc" hy_compile(tokenize("(assoc x y z)")) From c241236bf5a2e842040fbdaa1738b97708cac569 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 22 Apr 2013 00:54:58 +0200 Subject: [PATCH 38/41] Remove useless class attribute Signed-off-by: Julien Danjou --- hy/core/mangles.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hy/core/mangles.py b/hy/core/mangles.py index 36b766c..d60a8b3 100644 --- a/hy/core/mangles.py +++ b/hy/core/mangles.py @@ -64,8 +64,6 @@ class FunctionMangle(HoistableMangle): an anon function "((fn [] ...))", or using it as an arg in a call) """ - hoistable = ["fn"] - # ^^ we're just looking for functions ignore = ["def", "decorate_with", "setf", "setv", "foreach", "do"] # ^^ these guys don't affect us, really. From a9a67e9418361f026eac81bba134017ce43b9684 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 22 Apr 2013 22:46:43 +0200 Subject: [PATCH 39/41] Simplify compile Signed-off-by: Julien Danjou --- hy/compiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index 223242e..c1d6178 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -135,9 +135,9 @@ class HyASTCompiler(object): def compile(self, tree): try: - for _type in _compile_table: - if type(tree) == _type: - return _compile_table[_type](self, tree) + _type = type(tree) + if _type in _compile_table: + return _compile_table[_type](self, tree) except HyCompileError: # compile calls compile, so we're going to have multiple raise # nested; so let's re-raise this exception, let's not wrap it in From 109e8cb147799e09ce17919ce8d97dec326fc8d5 Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Mon, 22 Apr 2013 21:47:43 -0400 Subject: [PATCH 40/41] sorting imports by length --- hy/compiler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index c1d6178..bd74021 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -23,24 +23,24 @@ from hy.errors import HyError -from hy.models.expression import HyExpression -from hy.models.integer import HyInteger from hy.models.lambdalist import HyLambdaListKeyword -from hy.models.float import HyFloat +from hy.models.expression import HyExpression +from hy.models.keyword import HyKeyword +from hy.models.integer import HyInteger from hy.models.complex import HyComplex from hy.models.string import HyString from hy.models.symbol import HySymbol +from hy.models.float import HyFloat from hy.models.list import HyList from hy.models.dict import HyDict -from hy.models.keyword import HyKeyword from hy.util import flatten_literal_list, str_type, temporary_attribute_value from collections import defaultdict +import traceback import codecs import ast import sys -import traceback class HyCompileError(HyError): From 52ebcfae8c65f9d7ad7d1c10653aab07125c7605 Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Mon, 22 Apr 2013 21:49:19 -0400 Subject: [PATCH 41/41] removing the dunder. that was a bad move. --- hy/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/compiler.py b/hy/compiler.py index bd74021..a112d3a 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -84,7 +84,7 @@ def ast_str(foobar): enc = codecs.getencoder('punycode') foobar, _ = enc(foobar) - return "__hy_%s" % (str(foobar).replace("-", "_")) + return "hy_%s" % (str(foobar).replace("-", "_")) def builds(_type):