Merge branch 'master' into pr/441

AUTHORS

@@ -35,3 +35,5 @@
 * Thom Neale <twneale@gmail.com>
 * Tuukka Turto <tuukka.turto@oktaeder.net>
 * Vasudev Kamath <kamathvasudev@gmail.com>
+* Yuval Langer <yuval.langer@gmail.com>
+* Fatih Kadir Akın <fka@fatihak.in>

bin/hy2py

@@ -1,24 +1,39 @@
 #!/usr/bin/env python
 from __future__ import print_function
+from hy.importer import (import_file_to_ast, import_file_to_hst)
 
-from hy.importer import (import_file_to_ast, import_file_to_module,
-                         import_file_to_hst)
+import argparse
+import sys
 
 import astor.codegen
-import sys
 
 module_name = "<STDIN>"
 
-hst = import_file_to_hst(sys.argv[1])
-print(str(hst).encode("utf-8"))
-print("")
-print("")
-_ast = import_file_to_ast(sys.argv[1], module_name)
-print("")
-print("")
-print(astor.dump(_ast).encode("utf-8"))
-print("")
-print("")
-print(astor.codegen.to_source(_ast).encode("utf-8"))
+parser = argparse.ArgumentParser(
+    prog="hy2py",
+    usage="%(prog)s [options] FILE",
+    formatter_class=argparse.RawDescriptionHelpFormatter)
+parser.add_argument("--with-source", "-s", action="store_true",
+                    help="Show the parsed source structure")
+parser.add_argument("--with-ast", "-a", action="store_true",
+                    help="Show the generated AST")
+parser.add_argument("--without-python", "-np", action="store_true",
+                    help="Do not show the python code generated from the AST")
+parser.add_argument('args', nargs=argparse.REMAINDER, help=argparse.SUPPRESS)
 
-import_file_to_module(module_name, sys.argv[1])
+options = parser.parse_args(sys.argv[1:])
+
+if options.with_source:
+    hst = import_file_to_hst(options.args[0])
+    print(str(hst).encode("utf-8"))
+    print()
+    print()
+
+_ast = import_file_to_ast(options.args[0], module_name)
+if options.with_ast:
+    print(astor.dump(_ast).encode("utf-8"))
+    print()
+    print()
+
+if not options.without_python:
+    print(astor.codegen.to_source(_ast).encode("utf-8"))

docs/hacking.rst

@@ -97,7 +97,7 @@ core team. Additional review is clearly welcome, but we need a minimum of
 2 signoffs for any change.
 
 If a core member is sending in a PR, please find 2 core members that don't
-include them PR submitter. The idea here is that one can work with the PR
+include the PR submitter. The idea here is that one can work with the PR
 author, and a second acks the entire change set.
 
 If the change is adding documentation, feel free to just merge after one

docs/language/api.rst

@@ -36,7 +36,7 @@ languages.
 Builtins
 ========
 
-Hy features a number special forms that are used to help generate
+Hy features a number of special forms that are used to help generate
 correct Python AST. The following are "special" forms, which may have
 behavior that's slightly unexpected in some situations.
 
@@ -89,7 +89,7 @@ The following code demonstrates this:
 ---
 
 `->>` or `threading tail macro` is similar to `threading macro` but instead of
-inserting each expression into the next expression’s first argument place it
+inserting each expression into the next expression’s first argument place, it
 appends it as the last argument. The following code demonstrates this:
 
 .. code-block:: clj
@@ -110,6 +110,7 @@ Usage: `(apply fn-name [args] [kwargs])`
 Examples:
 
 .. code-block:: clj
+
     (defn thunk []
       "hy there")
 
@@ -282,7 +283,7 @@ do / progn
 the `do` and `progn` forms are used to evaluate each of their arguments and
 return the last one. Return values from every other than the last argument are
 discarded. It can be used in `lambda` or `list-comp` to perform more complex
-logic as show by one of the examples.
+logic as shown by one of the examples.
 
 Some example usage:
 
@@ -429,7 +430,7 @@ defmacro
 `defmacro` is used to define macros. The general format is
 `(defmacro [parameters] expr)`.
 
-Following example defines a macro that can be used to swap order of elements in
+The following example defines a macro that can be used to swap order of elements in
 code, allowing the user to write code in infix notation, where operator is in
 between the operands.
 
@@ -1166,7 +1167,7 @@ yield
 The generator is iterable and therefore can be used in loops, list
 comprehensions and other similar constructs.
 
-Especially the second example shows how generators can be used to generate
+The function random-numbers shows how generators can be used to generate
 infinite series without consuming infinite amount of memory.
 
 .. code-block:: clj

docs/language/core.rst

@@ -9,7 +9,7 @@ Core Functions
 .. _is-coll-fn:
 
 coll?
-----
+-----
 
 .. versionadded:: 0.9.13
 

docs/quickstart.rst

@@ -2,20 +2,20 @@
 Quickstart
 ==========
 
-.. image:: _static/cuddles.png
+.. image:: _static/cuddles-transparent-small.png
    :alt: Karen Rustard's Cuddles
 
-(thanks to Karen Rustad for Cuddles!)
+(Thanks to Karen Rustad for Cuddles!)
 
 
-HOW TO GET HY REAL FAST:
+**HOW TO GET HY REAL FAST**:
 
-1. create a `Python virtual environment
+1. Create a `Virtual Python Environment
    <https://pypi.python.org/pypi/virtualenv>`_
-2. activate your Python virtual environment
-3. ``pip install hy``
-4. start a REPL with ``hy``
-5. type stuff in the REPL::
+2. Activate your Virtual Python Environment
+3. Install `hy from PyPI <https://pypi.python.org/pypi/hy>`_ with ``pip install hy``
+4. Start a REPL with ``hy``
+5. Type stuff in the REPL::
 
        => (print "Hy!")
        Hy!
@@ -25,20 +25,19 @@ HOW TO GET HY REAL FAST:
 
        etc
 
-6. hit CTRL-D when you're done
+6. Hit CTRL-D when you're done
 
 OMG! That's amazing! I want to write a hy program.
 
-7. open up an elite programming editor
-8. type::
+7. Open up an elite programming editor and type::
 
-       (print "i was going to code in python syntax, but then i got hy")
+       (print "I was going to code in python syntax, but then I got hy.")
 
-9. save as ``test_program_of_awesome.hy``
-10. run::
+8. Save as ``awesome.hy``
+9. And run your first Hy program::
 
-        hy test_program_of_awesome.hy
+        hy awesome.hy
 
-11. take a deep breath so as to not hyperventilate
-12. smile villainously and sneak off to your hydeaway and do
+10. Take a deep breath so as to not hyperventilate
+11. Smile villainously and sneak off to your hydeaway and do
     unspeakable things

docs/tutorial.rst

@@ -406,7 +406,7 @@ The same thing in Hy::
   ... 
   [3, 2, 1, 4]
 
-See how we use kwapply to handle the fancy pssing? :)
+See how we use kwapply to handle the fancy passing? :)
 
 There's also a dictionary-style keyword arguments construction that
 looks like:

hy/_compat.py

@@ -38,6 +38,7 @@ except ImportError:
                        (x >> 24) & 0xff]))
 import sys
 
+PY27 = sys.version_info >= (2, 7)
 PY3 = sys.version_info[0] >= 3
 PY33 = sys.version_info >= (3, 3)
 PY34 = sys.version_info >= (3, 4)

hy/cmdline.py

@@ -214,8 +214,8 @@ def run_repl(hr=None, spy=False):
     return 0
 
 
-def run_icommand(source):
-    hr = HyREPL()
+def run_icommand(source, spy=False):
+    hr = HyREPL(spy)
     hr.runsource(source, filename='<input>', symbol='single')
     return run_repl(hr)
 
@@ -270,7 +270,7 @@ def cmdline_handler(scriptname, argv):
 
     if options.icommand:
         # User did "hy -i ..."
-        return run_icommand(options.icommand)
+        return run_icommand(options.icommand, spy=options.spy)
 
     if options.args:
         if options.args[0] == "-":

hy/compiler.py

@@ -39,7 +39,7 @@ from hy.errors import HyCompileError, HyTypeError
 
 import hy.macros
 from hy.macros import require, macroexpand
-from hy._compat import str_type, long_type, PY33, PY3, PY34
+from hy._compat import str_type, long_type, PY27, PY33, PY3, PY34
 import hy.importer
 
 import traceback
@@ -1265,41 +1265,114 @@ class HyASTCompiler(object):
                          ctx=ast.Load())
         return ret
 
+    def _compile_generator_iterables(self, trailers):
+        """Helper to compile the "trailing" parts of comprehensions:
+        generators and conditions"""
+
+        generators = trailers.pop(0)
+
+        cond = self.compile(trailers.pop(0)) if trailers != [] else Result()
+
+        gen_it = iter(generators)
+        paired_gens = zip(gen_it, gen_it)
+
+        gen_res = Result()
+        gen = []
+        for target, iterable in paired_gens:
+            comp_target = self.compile(target)
+            target = self._storeize(comp_target)
+            gen_res += self.compile(iterable)
+            gen.append(ast.comprehension(
+                target=target,
+                iter=gen_res.force_expr,
+                ifs=[]))
+
+        if cond.expr:
+            gen[-1].ifs.append(cond.expr)
+
+        return gen_res + cond, gen
+
     @builds("list_comp")
     @checkargs(min=2, max=3)
     def compile_list_comprehension(self, expr):
         # (list-comp expr (target iter) cond?)
         expr.pop(0)
         expression = expr.pop(0)
-        tar_it = iter(expr.pop(0))
-        targets = zip(tar_it, tar_it)
 
-        cond = self.compile(expr.pop(0)) if expr != [] else Result()
-
-        generator_res = Result()
-        generators = []
-        for target, iterable in targets:
-            comp_target = self.compile(target)
-            target = self._storeize(comp_target)
-            generator_res += self.compile(iterable)
-            generators.append(ast.comprehension(
-                target=target,
-                iter=generator_res.force_expr,
-                ifs=[]))
-
-        if cond.expr:
-            generators[-1].ifs.append(cond.expr)
+        gen_res, gen = self._compile_generator_iterables(expr)
 
         compiled_expression = self.compile(expression)
-        ret = compiled_expression + generator_res + cond
+        ret = compiled_expression + gen_res
         ret += ast.ListComp(
             lineno=expr.start_line,
             col_offset=expr.start_column,
             elt=compiled_expression.force_expr,
-            generators=generators)
+            generators=gen)
 
         return ret
 
+    @builds("set_comp")
+    @checkargs(min=2, max=3)
+    def compile_set_comprehension(self, expr):
+        if PY27:
+            ret = self.compile_list_comprehension(expr)
+            expr = ret.expr
+            ret.expr = ast.SetComp(
+                lineno=expr.lineno,
+                col_offset=expr.col_offset,
+                elt=expr.elt,
+                generators=expr.generators)
+
+            return ret
+
+        expr[0] = HySymbol("list_comp").replace(expr[0])
+        expr = HyExpression([HySymbol("set"), expr]).replace(expr)
+        return self.compile(expr)
+
+    @builds("dict_comp")
+    @checkargs(min=3, max=4)
+    def compile_dict_comprehension(self, expr):
+        if PY27:
+            expr.pop(0)  # dict-comp
+            key = expr.pop(0)
+            value = expr.pop(0)
+
+            gen_res, gen = self._compile_generator_iterables(expr)
+
+            compiled_key = self.compile(key)
+            compiled_value = self.compile(value)
+            ret = compiled_key + compiled_value + gen_res
+            ret += ast.DictComp(
+                lineno=expr.start_line,
+                col_offset=expr.start_column,
+                key=compiled_key.force_expr,
+                value=compiled_value.force_expr,
+                generators=gen)
+
+            return ret
+
+        # In Python 2.6, turn (dict-comp key value [foo]) into
+        # (dict (list-comp (, key value) [foo]))
+
+        expr[0] = HySymbol("list_comp").replace(expr[0])
+        expr[1:3] = [HyExpression(
+            [HySymbol(",")] +
+            expr[1:3]
+        ).replace(expr[1])]
+        expr = HyExpression([HySymbol("dict"), expr]).replace(expr)
+        return self.compile(expr)
+
+    @builds("genexpr")
+    def compile_genexpr(self, expr):
+        ret = self.compile_list_comprehension(expr)
+        expr = ret.expr
+        ret.expr = ast.GeneratorExp(
+            lineno=expr.lineno,
+            col_offset=expr.col_offset,
+            elt=expr.elt,
+            generators=expr.generators)
+        return ret
+
     @builds("apply")
     @checkargs(min=1, max=3)
     def compile_apply_expression(self, expr):

hy/contrib/loop.hy

@@ -1,6 +1,7 @@
 ;;; Hy tail-call optimization
 ;;
 ;; Copyright (c) 2014 Clinton Dreisbach <clinton@dreisbach.us>
+;; Copyright (c) 2014 Paul R. Tagliamonte <tag@pault.ag>
 ;;
 ;; Permission is hereby granted, free of charge, to any person obtaining a
 ;; copy of this software and associated documentation files (the "Software"),
@@ -55,7 +56,24 @@
                 (recursive-replace old-term new-term term)]
                [True term]) [term body])))
 
-(defmacro loop [bindings &rest body]
+
+(defmacro/g! fnr [signature &rest body]
+  (let [[new-body (recursive-replace 'recur g!recur-fn body)]]
+    `(do
+      (import [hy.contrib.loop [--trampoline--]])
+      (with-decorator
+        --trampoline--
+        (def ~g!recur-fn (fn [~@signature] ~@new-body)))
+      ~g!recur-fn)))
+
+
+(defmacro defnr [name lambda-list &rest body]
+  (if (not (= (type name) HySymbol))
+    (macro-error name "defnr takes a name as first argument"))
+  `(setv ~name (fnr ~lambda-list ~@body)))
+
+
+(defmacro/g! loop [bindings &rest body]
   ;; Use inside functions like so:
   ;; (defun factorial [n]
   ;;   (loop [[i n]
@@ -67,13 +85,7 @@
   ;; If recur is used in a non-tail-call position, None is returned, which
   ;; causes chaos. Fixing this to detect if recur is in a tail-call position
   ;; and erroring if not is a giant TODO.
-  (with-gensyms [recur-fn]
-    (let [[fnargs (map (fn [x] (first x)) bindings)]
-          [initargs (map second bindings)]
-          [new-body (recursive-replace 'recur recur-fn body)]]
-      `(do
-        (import [hy.contrib.loop [--trampoline--]])
-        (def ~recur-fn
-          (--trampoline-- (fn [~@fnargs]
-                            ~@new-body)))
-        (~recur-fn ~@initargs)))))
+  (let [[fnargs (map (fn [x] (first x)) bindings)]
+        [initargs (map second bindings)]]
+    `(do (defnr ~g!recur-fn [~@fnargs] ~@body)
+         (~g!recur-fn ~@initargs))))

hy/lex/parser.py

@@ -242,14 +242,19 @@ def t_identifier(p):
     if obj.startswith("&"):
         return HyLambdaListKeyword(obj)
 
-    if obj.startswith("*") and obj.endswith("*") and obj not in ("*", "**"):
-        obj = obj[1:-1].upper()
+    def mangle(p):
+        if p.startswith("*") and p.endswith("*") and p not in ("*", "**"):
+            p = p[1:-1].upper()
 
-    if "-" in obj and obj != "-":
-        obj = obj.replace("-", "_")
+        if "-" in p and p != "-":
+            p = p.replace("-", "_")
 
-    if obj.endswith("?") and obj != "?":
-        obj = "is_%s" % (obj[:-1])
+        if p.endswith("?") and p != "?":
+            p = "is_%s" % (p[:-1])
+
+        return p
+
+    obj = ".".join([mangle(part) for part in obj.split(".")])
 
     return HySymbol(obj)
 

tests/lex/test_lex.py

@@ -266,3 +266,39 @@ def test_lex_comment_382():
     """Ensure that we can tokenize sources with a comment at the end"""
     entry = tokenize("foo ;bar\n;baz")
     assert entry == [HySymbol("foo")]
+
+
+def test_lex_mangling_star():
+    """Ensure that mangling starred identifiers works according to plan"""
+    entry = tokenize("*foo*")
+    assert entry == [HySymbol("FOO")]
+    entry = tokenize("*")
+    assert entry == [HySymbol("*")]
+    entry = tokenize("*foo")
+    assert entry == [HySymbol("*foo")]
+
+
+def test_lex_mangling_hyphen():
+    """Ensure that hyphens get translated to underscores during mangling"""
+    entry = tokenize("foo-bar")
+    assert entry == [HySymbol("foo_bar")]
+    entry = tokenize("-")
+    assert entry == [HySymbol("-")]
+
+
+def test_lex_mangling_qmark():
+    """Ensure that identifiers ending with a question mark get mangled ok"""
+    entry = tokenize("foo?")
+    assert entry == [HySymbol("is_foo")]
+    entry = tokenize("?")
+    assert entry == [HySymbol("?")]
+    entry = tokenize("im?foo")
+    assert entry == [HySymbol("im?foo")]
+    entry = tokenize(".foo?")
+    assert entry == [HySymbol(".is_foo")]
+    entry = tokenize("foo.bar?")
+    assert entry == [HySymbol("foo.is_bar")]
+    entry = tokenize("foo?.bar")
+    assert entry == [HySymbol("is_foo.bar")]
+    entry = tokenize(".foo?.bar.baz?")
+    assert entry == [HySymbol(".is_foo.bar.is_baz")]

tests/native_tests/language.hy

@@ -525,7 +525,7 @@
     (assert (= x 3))))
 
 
-(defn test-comprehensions []
+(defn test-list-comprehensions []
   "NATIVE: test list comprehensions"
   (assert (= (list-comp (* x 2) (x (range 2))) [0 2]))
   (assert (= (list-comp (* x 2) (x (range 4)) (% x 2)) [2 6]))
@@ -536,6 +536,41 @@
   (assert (= (list-comp j (j [1 2])) [1 2])))
 
 
+(defn test-set-comprehensions []
+  "NATIVE: test set comprehensions"
+  (assert (instance? set (set-comp x [x (range 2)])))
+  (assert (= (set-comp (* x 2) (x (range 2))) (set [0 2])))
+  (assert (= (set-comp (* x 2) (x (range 4)) (% x 2)) (set [2 6])))
+  (assert (= (set-comp (* y 2) ((, x y) (.items {"1" 1 "2" 2})))
+             (set [2 4])))
+  (assert (= (set-comp (, x y) (x (range 2) y (range 2)))
+             (set [(, 0 0) (, 0 1) (, 1 0) (, 1 1)])))
+  (assert (= (set-comp j (j [1 2])) (set [1 2]))))
+
+
+(defn test-dict-comprehensions []
+  "NATIVE: test dict comprehensions"
+  (assert (instance? dict (dict-comp x x [x (range 2)])))
+  (assert (= (dict-comp x (* x 2) (x (range 2))) {1 2 0 0}))
+  (assert (= (dict-comp x (* x 2) (x (range 4)) (% x 2)) {3 6 1 2}))
+  (assert (= (dict-comp x (* y 2) ((, x y) (.items {"1" 1 "2" 2})))
+             {"2" 4 "1" 2}))
+  (assert (= (dict-comp (, x y) (+ x y) (x (range 2) y (range 2)))
+             {(, 0 0) 0 (, 1 0) 1 (, 0 1) 1 (, 1 1) 2})))
+
+
+(defn test-generator-expressions []
+  "NATIVE: test generator expressions"
+  (assert (not (instance? list (genexpr x [x (range 2)]))))
+  (assert (= (list (genexpr (* x 2) (x (range 2)))) [0 2]))
+  (assert (= (list (genexpr (* x 2) (x (range 4)) (% x 2))) [2 6]))
+  (assert (= (list (sorted (genexpr (* y 2) ((, x y) (.items {"1" 1 "2" 2})))))
+             [2 4]))
+  (assert (= (list (genexpr (, x y) (x (range 2) y (range 2))))
+             [(, 0 0) (, 0 1) (, 1 0) (, 1 1)]))
+  (assert (= (list (genexpr j (j [1 2]))) [1 2])))
+
+
 (defn test-defn-order []
   "NATIVE: test defn evaluation order"
   (setv acc [])

tests/test_bin.py

@@ -76,6 +76,14 @@ def test_bin_hy_icmd():
     assert "figlet" in output
 
 
+def test_bin_hy_icmd_and_spy():
+    ret = run_cmd("hy -i \"(+ [] [])\" --spy", "(+ 1 1)")
+    assert ret[0] == 0
+    output = ret[1]
+
+    assert "([] + [])" in output
+
+
 def test_bin_hy_missing_file():
     ret = run_cmd("hy foobarbaz")
     assert ret[0] == 2
@@ -126,7 +134,7 @@ def test_hy2py():
         for f in filenames:
             if f.endswith(".hy"):
                 i += 1
-                ret = run_cmd("bin/hy2py " + os.path.join(dirpath, f))
+                ret = run_cmd("bin/hy2py -s -a " + os.path.join(dirpath, f))
                 assert ret[0] == 0, f
                 assert len(ret[1]) > 1, f
                 assert len(ret[2]) == 0, f