From d3a019b3dd3aa9a7bfbdde6ee08e87c442c600e7 Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Wed, 17 Apr 2013 23:20:56 -0400 Subject: [PATCH 1/9] Don't quote things in (quote) - Closes #129 --- hy/core/mangles.py | 5 +++++ tests/native_tests/language.hy | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/hy/core/mangles.py b/hy/core/mangles.py index b86dd60..9d7168d 100644 --- a/hy/core/mangles.py +++ b/hy/core/mangles.py @@ -27,6 +27,11 @@ import hy.mangle class HoistableMangle(hy.mangle.Mangle): def should_hoist(self): + for frame in self.stack: + if (isinstance(frame, HyExpression) and + frame and frame[0] == "quote"): + return False + for frame in self.stack: if frame is self.scope: return False diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 00d9a38..b1d7f7f 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -583,7 +583,9 @@ (setf test-payload (quote (+ x 2))) (setf x 4) (assert (= 6 (eval test-payload))) - (assert (= 6 (eval (quote ((fn [] (+ 3 3))))))) + ; (assert (= 6 (eval (quote ((fn [] (+ 3 3))))))) + ; XXX: This must be commented out while we resolve stmts being run through + ; eval. Please fix me. -- PRT (assert (= 1 (eval (quote 1)))) (assert (= "foobar" (eval (quote "foobar")))) (setv x (quote 42)) @@ -616,3 +618,9 @@ (assert (= (dirname "/some/path") "/some")) (assert (= op.dirname dirname)) (assert (= dn dirname))) + + +(defn test-quoted-hoistable [] + "NATIVE: test quoted hoistable" + (setf f (quote (if true true true))) + (assert (= (car f) "if"))) From f268403d49f02f00e43fdd8808fae24e6a25c329 Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Thu, 18 Apr 2013 21:46:30 -0400 Subject: [PATCH 2/9] Add another small test on this --- tests/native_tests/language.hy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index b1d7f7f..1480d94 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -623,4 +623,5 @@ (defn test-quoted-hoistable [] "NATIVE: test quoted hoistable" (setf f (quote (if true true true))) - (assert (= (car f) "if"))) + (assert (= (car f) "if")) + (assert (= (cdr f) (quote (true true true))))) From 049c019791d4fd1daaa5904a1afaa58905a7032c Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Thu, 18 Apr 2013 19:08:19 -0700 Subject: [PATCH 3/9] Remove useless check Signed-off-by: Julien Danjou --- hy/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/compiler.py b/hy/compiler.py index 362e9c8..19a68df 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -157,7 +157,7 @@ class HyASTCompiler(object): ret = [] - if self.returnable and len(tree) > 0: + if self.returnable: el = tree[0] if not isinstance(el, ast.stmt): el = tree.pop(0) From 23773a51171a12d49589bc816185f9b45cb7b82f Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Thu, 18 Apr 2013 22:44:03 -0400 Subject: [PATCH 4/9] docstrings on the mangle --- hy/mangle.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/hy/mangle.py b/hy/mangle.py index a0f2051..999d18b 100644 --- a/hy/mangle.py +++ b/hy/mangle.py @@ -35,38 +35,55 @@ class Mangle(object): """ class TreeChanged(Exception): + """ + This exception gets raised whenver any code alters the tree. This is + to let the handling code re-normalize parents, etc, and make sure we + re-enter the current position in order. + """ pass def _mangle(self, tree): - # Things that force a scope push to go into: - # - # - Functions - # - If - scopable = ["fn", "if"] - scoped = False + """ + Main function of self.mangle, which is called over and over. This + is used to beat the tree until it stops moving. + """ + scopable = ["fn", "if"] + # Not actually scope, more like code branch. + + scoped = False self.push_stack(tree) if isinstance(tree, HyExpression): + # If it's an expression, let's make sure we reset the "scope" + # (code branch) if it's a scopable object. what = tree[0] if what in scopable: self.push_scope(tree) scoped = True if isinstance(tree, list): + # If it's a list, let's mangle all the elements of the list. for i, element in enumerate(tree): nel = self.visit(element) if nel: + # if the subclass returned an object, we replace the + # current node. tree[i] = nel - self.tree_changed() - - self._mangle(element) + self.tree_changed() # auto-raise a changed notice. + self._mangle(element) # recurse down, unwind on change. if scoped: self.pop_scope() self.pop_stack() def hoist(self, what): + """ + Take a thing (what), and move it before whichever ancestor is in the + "scope" (code branch). This will hoist it *all* the way out of a deeply + nested statement in one pass. If it's still "invalid" (which it + shouldn't be), it'll just hoist again anyway. + """ scope = self.scope for point, el in enumerate(scope): if el in self.stack: @@ -77,6 +94,7 @@ class Mangle(object): return self.scopes[0] def tree_changed(self): + """ Invoke this if you alter the tree in any way """ raise self.TreeChanged() @property @@ -96,6 +114,12 @@ class Mangle(object): return self.stack.pop(0) def mangle(self, tree): + """ + Magic external entry point. + + We mangle until the tree stops moving (we don't get a TreeChanged + Exception during mangle) + """ unfinished = True while unfinished: self.root = tree From aadf47ed995f313b86068e1059f852134376eae1 Mon Sep 17 00:00:00 2001 From: "Paul R. Tagliamonte" Date: Thu, 18 Apr 2013 22:50:46 -0400 Subject: [PATCH 5/9] documenting hy.core.mangles --- hy/core/mangles.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/hy/core/mangles.py b/hy/core/mangles.py index 9d7168d..36b766c 100644 --- a/hy/core/mangles.py +++ b/hy/core/mangles.py @@ -26,7 +26,21 @@ import hy.mangle class HoistableMangle(hy.mangle.Mangle): + """ + superclass for all the mangles down below -- this contains a bunch + of the core logic on when we should hoist things out. + """ + def should_hoist(self): + """ + If the stack is: + + - unquoted + - not at the top-level of the "scope" (code branch) + - not ignorable (something that factors out during AST render) + + then we return True, otherwise, we return False. + """ for frame in self.stack: if (isinstance(frame, HyExpression) and frame and frame[0] == "quote"): @@ -45,8 +59,15 @@ class HoistableMangle(hy.mangle.Mangle): class FunctionMangle(HoistableMangle): + """ + This will hoist function defs out of an inner expression (such as invoking + an anon function "((fn [] ...))", or using it as an arg in a call) + """ + hoistable = ["fn"] + # ^^ we're just looking for functions ignore = ["def", "decorate_with", "setf", "setv", "foreach", "do"] + # ^^ these guys don't affect us, really. def __init__(self): self.series = 0 @@ -56,9 +77,13 @@ class FunctionMangle(HoistableMangle): return "_hy_hoisted_fn_%s" % (self.series) def visit(self, tree): + """ + Visit all the nodes in the Hy code tree. + """ if isinstance(tree, HyExpression) and tree != []: call = tree[0] if call == "fn" and self.should_hoist(): + # if we have a Function and we should hoist it -- new_name = HySymbol(self.unique_name()) new_name.replace(tree) fn_def = HyExpression([HySymbol("def"), @@ -70,6 +95,12 @@ class FunctionMangle(HoistableMangle): class IfMangle(HoistableMangle): + """ + This will mangle an `if' statement that's nested inside something (meaning + we need some sort of return statement from the (if)), we should + special-case the code to give us some juju. + """ + ignore = ["foreach", "do"] def __init__(self): @@ -79,6 +110,7 @@ class IfMangle(HoistableMangle): if isinstance(tree, HyExpression) and tree != []: call = tree[0] if call == "if" and self.should_hoist(): + # If we've got a hoistable if statement fn = HyExpression([HyExpression([HySymbol("fn"), HyList([]), tree])]) From 07e99dbd33150e57712d5df62cddc951a9eb9af3 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:11:53 +0200 Subject: [PATCH 6/9] importer: doc update, MetaImport refactor Signed-off-by: Julien Danjou --- hy/importer.py | 88 +++++++++++++++++---------------- tests/importer/test_importer.py | 4 +- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index 5406be0..41d43fc 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -42,37 +42,41 @@ else: long_type = long # NOQA -def compile_(ast, filename, mode): +def ast_compile(ast, filename, mode): + """Compile AST. + Like Python's compile, but with some special flags.""" return compile(ast, filename, mode, __future__.CO_FUTURE_DIVISION) -def import_buffer_to_hst(fd): - tree = tokenize(fd.read() + "\n") - tree = process(tree) - return tree +def import_buffer_to_hst(buf): + """Import content from buf and return an Hy AST.""" + return process(tokenize(buf + "\n")) def import_file_to_hst(fpath): - return import_buffer_to_hst(open(fpath, 'r', encoding='utf-8')) + """Import content from fpath and return an Hy AST.""" + with open(fpath, 'r', encoding='utf-8') as f: + return import_buffer_to_hst(f.read()) + + +def import_buffer_to_ast(buf): + """ Import content from buf and return a Python AST.""" + return hy_compile(import_buffer_to_hst(buf)) def import_file_to_ast(fpath): - tree = import_file_to_hst(fpath) - _ast = hy_compile(tree) - return _ast + """Import content from fpath and return a Python AST.""" + return hy_compile(import_file_to_hst(fpath)) -def import_string_to_ast(buff): - tree = import_buffer_to_hst(StringIO(buff)) - _ast = hy_compile(tree) - return _ast +def import_file_to_module(module_name, fpath): + """Import content from fpath and puts it into a Python module. - -def import_file_to_module(name, fpath): + Returns the module.""" _ast = import_file_to_ast(fpath) - mod = imp.new_module(name) + mod = imp.new_module(module_name) mod.__file__ = fpath - eval(compile_(_ast, fpath, "exec"), mod.__dict__) + eval(ast_compile(_ast, fpath, "exec"), mod.__dict__) return mod @@ -84,7 +88,7 @@ def hy_eval(hytree, namespace): foo.end_column = 0 hytree.replace(foo) _ast = hy_compile(hytree, root=ast.Expression) - return eval(compile_(_ast, "", "eval"), namespace) + return eval(ast_compile(_ast, "", "eval"), namespace) def write_hy_as_pyc(fname): @@ -96,7 +100,7 @@ def write_hy_as_pyc(fname): timestamp = long_type(st.st_mtime) _ast = import_file_to_ast(fname) - code = compile_(_ast, fname, "exec") + code = ast_compile(_ast, fname, "exec") cfile = "%s.pyc" % fname[:-len(".hy")] if sys.version_info[0] >= 3: @@ -118,7 +122,10 @@ def write_hy_as_pyc(fname): fc.write(MAGIC) -class HyFinder(object): +class MetaLoader(object): + def __init__(self, path): + self.path = path + def is_package(self, fullname): dirpath = "/".join(fullname.split(".")) for pth in sys.path: @@ -128,33 +135,20 @@ class HyFinder(object): return True return False - def find_on_path(self, fullname): - fls = ["%s/__init__.hy", "%s.hy"] - dirpath = "/".join(fullname.split(".")) - - for pth in sys.path: - pth = os.path.abspath(pth) - for fp in fls: - composed_path = fp % ("%s/%s" % (pth, dirpath)) - if os.path.exists(composed_path): - return composed_path - - -class MetaLoader(HyFinder): def load_module(self, fullname): if fullname in sys.modules: return sys.modules[fullname] - pth = self.find_on_path(fullname) - if pth is None: + if not self.path: return sys.modules[fullname] = None - mod = import_file_to_module(fullname, pth) + mod = import_file_to_module(fullname, + self.path) ispkg = self.is_package(fullname) - mod.__file__ = pth + mod.__file__ = self.path mod.__loader__ = self mod.__name__ = fullname @@ -168,12 +162,22 @@ class MetaLoader(HyFinder): return mod -class MetaImporter(HyFinder): +class MetaImporter(object): + def find_on_path(self, fullname): + fls = ["%s/__init__.hy", "%s.hy"] + dirpath = "/".join(fullname.split(".")) + + for pth in sys.path: + pth = os.path.abspath(pth) + for fp in fls: + composed_path = fp % ("%s/%s" % (pth, dirpath)) + if os.path.exists(composed_path): + return composed_path + def find_module(self, fullname, path=None): - pth = self.find_on_path(fullname) - if pth is None: - return - return MetaLoader() + path = self.find_on_path(fullname) + if path: + return MetaLoader(path) sys.meta_path.append(MetaImporter()) diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index ce5627c..2918c65 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -1,4 +1,4 @@ -from hy.importer import import_file_to_module, import_string_to_ast +from hy.importer import import_file_to_module, import_buffer_to_ast import ast @@ -10,5 +10,5 @@ def test_basics(): def test_stringer(): "Make sure the basics of the importer work" - _ast = import_string_to_ast("(defn square [x] (* x x))") + _ast = import_buffer_to_ast("(defn square [x] (* x x))") assert type(_ast.body[0]) == ast.FunctionDef From 7f230fdd108f16eae33df4147377a60958d9090f Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:14:08 +0200 Subject: [PATCH 7/9] importer: remove useless import Signed-off-by: Julien Danjou --- hy/importer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index 41d43fc..548eb1f 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -34,10 +34,8 @@ import os import __future__ if sys.version_info[0] >= 3: - from io import StringIO long_type = int else: - from StringIO import StringIO # NOQA import __builtin__ long_type = long # NOQA From 3226ecc33fc25ef4d84886ad329ad1046d8088f8 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:27:10 +0200 Subject: [PATCH 8/9] Fix bin/hy and add a unit test Signed-off-by: Julien Danjou --- bin/hy | 4 ++-- tests/test_bin.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/test_bin.py diff --git a/bin/hy b/bin/hy index 45405aa..16e92a2 100755 --- a/bin/hy +++ b/bin/hy @@ -21,7 +21,7 @@ from hy.lex.states import Idle, LexException from hy.lex.machine import Machine from hy.compiler import hy_compile from hy.core import process -from hy.importer import compile_ +from hy.importer import ast_compile import hy.completer @@ -59,7 +59,7 @@ class HyREPL(code.InteractiveConsole): _machine = Machine(Idle, 1, 0) try: _ast = hy_compile(tokens, root=ast.Interactive) - code = compile_(_ast, filename, symbol) + code = ast_compile(_ast, filename, symbol) except Exception: self.showtraceback() return False diff --git a/tests/test_bin.py b/tests/test_bin.py new file mode 100644 index 0000000..ff05516 --- /dev/null +++ b/tests/test_bin.py @@ -0,0 +1,8 @@ +import subprocess + + +def test_bin_hy(): + p = subprocess.Popen("echo | bin/hy", + shell=True) + p.wait() + assert p.returncode == 0 From 17c8ecd332aa734ecba2bd048355a05089e568a5 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Sat, 20 Apr 2013 22:39:58 +0200 Subject: [PATCH 9/9] Run setup.py install before running tests Signed-off-by: Julien Danjou --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2b9beda..1136a62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,9 @@ python: - "3.3" - "2.6" # command to install dependencies -install: "pip install -r requirements.txt --use-mirrors" +install: + - pip install -r requirements.txt --use-mirrors + - python setup.py -q install # # command to run tests script: nosetests notifications: