Merge branch 'master' into pr/397

Conflicts:
	hy/importer.py

AUTHORS

@@ -21,3 +21,17 @@
 * Henrique Carvalho Alves <hcarvalhoalves@gmail.com>
 * Joe Hakim Rahme <joehakimrahme@gmail.com>
 * Kenan Bölükbaşı <kenanbolukbasi@gmail.com>
+* Abhishek L <abhishek.lekshmanan@gmail.com>
+* Christopher Browne <cbbrowne@ca.afilias.info>
+* Clinton N. Dreisbach <crnixon@gmail.com>
+* D. Joe <deejoe+castanea@etrumeus.com>
+* Duncan McGreggor <duncan.mcgreggor@rackspace.com>
+* E. Anders Lannerback <anders@lannerback.net>
+* Jack <jackjrabbit+github@gmail.com>
+* Johan Euphrosine <proppy@google.com>
+* Kevin Zita <kzita@kent.edu>
+* Matt Fenwick <mfenwick100@gmail.com>
+* Sean B. Palmer <sean@miscoranda.com>
+* Thom Neale <twneale@gmail.com>
+* Tuukka Turto <tuukka.turto@oktaeder.net>
+* Vasudev Kamath <kamathvasudev@gmail.com>

NEWS

@@ -1,9 +1,75 @@
 Changes from Hy 0.9.11
 
-    [ Misc. Fixes ]
+   tl;dr:
-    [ Syntax Fixes ]
+
+    0.9.12 comes with some massive changes,
+    We finally took the time to implement gensym, as well as a few
+    other bits that help macro writing. Check the changelog for
+    what exactly was added. 
+    
+    The biggest feature, Reader Macros, landed later
+    in the cycle, but were big enough to warrent a release on it's
+    own. A huge thanks goes to Foxboron for implementing them
+    and a massive hug goes out to olasd for providing ongoing
+    reviews during the development.
+    
+    Welcome to the new Hy contributors, Henrique Carvalho Alves,
+    Kevin Zita and Kenan Bölükbaşı. Thanks for your work so far,
+    folks!
+    
+    Hope y'all enjoy the finest that 2013 has to offer,
+      - Hy Society
+
+
+    * Special thanks goes to Willyfrog, Foxboron and theanalyst for writing
+      0.9.12's NEWS. Thanks, y'all! (PT)
+
+
     [ Language Changes ]
-     * Translate foo? -> is_foo, for better Python interop. (PT)
+    * Translate foo? -> is_foo, for better Python interop. (PT)
+    * Reader Macros!
+    * Operators + and * now can work without arguments
+    * Define kwapply as a macro
+    * Added apply as a function
+    * Instant symbol generation with gensym
+    * Allow macros to return None
+    * Add a method for casting into byte string or unicode depending on python version
+    * flatten function added to language
+    * Add a method for casting into byte string or unicode depending on python version
+    * Added type coercing to the right integer for the platform
+
+
+    [ Misc. Fixes ]
+    * Added information about core team members
+    * Documentation fixed and extended
+    * Add astor to install_requires to fix hy --spy failing on hy 0.9.11.
+    * Convert stdout and stderr to UTF-8 properly in the run_cmd helper.
+    * Update requirements.txt and setup.py to use rply upstream.
+    * tryhy link added in documentation and README
+    * Command line options documented
+    * Adding support for coverage tests at coveralls.io
+    * Added info about tox, so people can use it prior to a PR
+    * Added the start of hacking rules
+    * Halting Problem removed from example as it was nonfree
+    * Fixed PyPI is now behind a CDN. The --use-mirrors option is deprecated.
+    * Badges for pypi version and downloads.
+
+
+    [ Syntax Fixes ]
+    * get allows multiple arguments
+
+
+    [ Bug Fixes ]
+    *  OSX: Fixes for readline Repl problem which caused HyREPL not allowing 'b'
+    * Fix REPL completions on OSX
+    *  Make HyObject.replace more resilient to prevent compiler breakage.
+
+
+    [ Contrib changes ]
+    * Anaphoric macros added to contrib
+    * Modified eg/twisted to follow the newer hy syntax
+    * Added (experimental) profile module
+
 
 Changes from Hy 0.9.10
 

docs/conf.py

@@ -28,7 +28,7 @@ import hy
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = []
+extensions = ['sphinx.ext.todo']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -67,7 +67,7 @@ release = hy.__version__
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ['_build', 'coreteam.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None

docs/hacking.rst

@@ -100,6 +100,9 @@ If a core member is sending in a PR, please find 2 core members that don't
 include them PR submitter. The idea here is that one can work with the PR
 author, and a second acks the entire change set.
 
+If the change is adding documentation, feel free to just merge after one
+ACK. We've got low coverage, so it'd be great to keep that barrier low.
+
 
 Core Team
 =========

docs/index.rst

@@ -19,17 +19,17 @@ Meet our mascot, "Cuddles":
 .. image:: http://fc07.deviantart.net/fs70/i/2013/138/f/0/cuddles_the_hacker_by_doctormo-d65l7lq.png
    :alt: Paul riding cuddles into the distance
 
-
-..   Our old ascii art mascot version
-..   Retained as an easter egg for those who read the docs via .rst!
 ..
-..   LET'S CUDDLEFISH
+   Our old ascii art mascot version
-..              ______
+   Retained as an easter egg for those who read the docs via .rst!
-..       _.----'#'  # '
+
-..     ,'  #'    ,#  ;
+     LET'S CUDDLEFISH
-..    ('   (w)  _,-'_/
+                ______
-..   /// / /'.____.'
+         _.----'#'  # '
-..   \|\||/
+       ,'  #'    ,#  ;
+      ('   (w)  _,-'_/
+     /// / /'.____.'
+     \|\||/
 
 
 You can try Hy `in your browser <https://try-hy.appspot.com>`_.

docs/language/api.rst

@@ -205,11 +205,12 @@ however is called only for every other value in the list.
     ;; assuming that (side-effect1) and (side-effect2) are functions and
     ;; collection is a list of numerical values
 
-    (for (x collection) (do
+    (for [x collection]
-      (side-effect1 x)
+      (do
-      (if (% x 2)
+        (side-effect1 x)
-        (continue))
+        (if (% x 2)
-      (side-effect2 x)))
+          (continue))
+        (side-effect2 x)))
 
 
 do / progn
@@ -357,6 +358,8 @@ Parameters may have following keywords in front of them:
         => (zig-zag-sum 1 2 3 4 5 6)
         -3
 
+.. _defmacro:
+
 defmacro
 --------
 
@@ -378,6 +381,43 @@ between the operands.
   => (infix (1 + 1))
   2
 
+.. _defmacro/g!:
+
+defmacro/g!
+------------
+
+.. versionadded:: 0.9.12
+
+`defmacro/g!` is a special version of `defmacro` that is used to
+automatically generate :ref:`gensym` for any symbol that
+starts with ``g!``.
+
+So ``g!a`` would become ``(gensym "a")``.
+
+.. seealso::
+
+   Section :ref:`using-gensym`
+
+defreader
+---------
+
+.. versionadded:: 0.9.12
+
+`defreader` defines a reader macro, enabling you to restructure or
+modify syntax.
+
+.. code-block:: clj
+
+    => (defreader ^ [expr] (print expr))
+    => #^(1 2 3 4)
+    (1 2 3 4)
+    => #^"Hello"
+    "Hello"
+
+.. seealso::
+
+    Section :ref:`Reader Macros <reader-macros>`
+
 del
 ---
 
@@ -441,52 +481,36 @@ first / car
 
 
 for
----
-
-`for` macro is used to build nested `foreach` loops. The macro takes two
-parameters, first being a vector specifying collections to iterate over and 
-variables to bind. The second parameter is a statement which is executed during
-each loop:
-
-.. code-block:: clj
-
-    (for [x iter y iter] stmt)
-
-    (foreach [x iter]
-      (foreach [y iter] stmt))
-
-
-foreach
 -------
 
-`foreach` is used to call a function for each element in a list or vector.
+`for` is used to call a function for each element in a list or vector.
 Results are discarded and None is returned instead. Example code iterates over
 collection and calls side-effect to each element in the collection:
 
 .. code-block:: clj
 
     ;; assuming that (side-effect) is a function that takes a single parameter
-    (foreach [element collection] (side-effect element))
+    (for [element collection] (side-effect element))
 
-    ;; foreach can have an optional else block
+    ;; for can have an optional else block
-    (foreach [element collection] (side-effect element)
+    (for [element collection] (side-effect element)
-             (else (side-effect-2)))
+         (else (side-effect-2)))
 
-The optional `else` block is executed only if the `foreach` loop terminates
+The optional `else` block is executed only if the `for` loop terminates
 normally. If the execution is halted with `break`, the `else` does not execute.
 
 .. code-block:: clj
 
-    => (foreach [element [1 2 3]] (if (< element 3)
+    => (for [element [1 2 3]] (if (< element 3)
-    ...                               (print element) 
+    ...                             (print element) 
-    ...                               (break))
+    ...                             (break))
     ...    (else (print "loop finished")))
     1
     2
 
-    => (foreach [element [1 2 3]] (if (< element 4)
+    => (for [element [1 2 3]] (if (< element 4)
-    ...                               (print element)
+    ...                             (print element)
-    ...                               (break))
+    ...                             (break))
     ...    (else (print "loop finished")))
     1
     2
@@ -494,6 +518,28 @@ normally. If the execution is halted with `break`, the `else` does not execute.
     loop finished
 
 
+.. _gensym:
+
+gensym
+------
+
+.. versionadded:: 0.9.12
+
+`gensym` form is used to generate a unique symbol to allow writing macros
+without accidental variable name clashes.
+
+.. code-block:: clj
+
+   => (gensym)
+   u':G_1235'
+
+   => (gensym "x")
+   u':x_1236'
+
+.. seealso::
+
+   Section :ref:`using-gensym`
+
 get
 ---
 
@@ -635,7 +681,7 @@ function is defined and passed to another function for filtering output.
     ...             {:name "Dave" :age 5}])
 
     => (defn display-people [people filter]
-    ...  (foreach [person people] (if (filter person) (print (:name person)))))
+    ...  (for [person people] (if (filter person) (print (:name person)))))
 
     => (display-people people (fn [person] (< (:age person) 25)))
     Alice
@@ -949,16 +995,18 @@ context to an argument or ignore it completely, as shown below:
 
 .. code-block:: clj
 
-    (with [arg (expr)] block)
+    (with [[arg (expr)]] block)
 
-    (with [(expr)] block)
+    (with [[(expr)]] block)
+
+    (with [[arg (expr)] [(expr)]] block)
 
 The following example will open file `NEWS` and print its content on screen. The
 file is automatically closed after it has been processed.
 
 .. code-block:: clj
 
-    (with [f (open "NEWS")] (print (.read f)))
+    (with [[f (open "NEWS")]] (print (.read f)))
 
 
 with-decorator
@@ -983,6 +1031,35 @@ values that are incremented by 1. When decorated `addition` is called with value
     4
 
 
+.. _with-gensyms:
+
+with-gensyms
+-------------
+
+.. versionadded:: 0.9.12
+
+`with-gensym` form is used to generate a set of :ref:`gensym` for use
+in a macro.
+
+.. code-block:: clojure
+
+   (with-gensyms [a b c]
+     ...)
+
+expands to:
+
+.. code-block:: clojure
+
+   (let [[a (gensym)
+         [b (gensym)
+         [c (gensym)]]
+     ...)
+
+.. seealso::
+
+   Section :ref:`using-gensym`
+
+
 yield
 -----
 
@@ -996,7 +1073,7 @@ infinite series without consuming infinite amount of memory.
 .. code-block:: clj
 
     => (defn multiply [bases coefficients]
-    ...  (foreach [(, base coefficient) (zip bases coefficients)]
+    ...  (for [[(, base coefficient) (zip bases coefficients)]]
     ...   (yield (* base coefficient))))
 
     => (multiply (range 5) (range 5))

docs/language/cli.rst

@@ -32,6 +32,12 @@ Command line options
 
    .. versionadded:: 0.9.11
 
+.. cmdoption:: --show-tracebacks
+
+   Print extended tracebacks for Hy exceptions.
+
+   .. versionadded:: 0.9.12
+
 .. cmdoption:: -v
 
    Print the Hy version number and exit.

docs/language/core.rst

@@ -242,6 +242,36 @@ Raises ``TypeError`` if ``(not (numeric? x))``.
    => (neg? 0)
    False
 
+
+.. _nil?-fn:
+
+nil?
+-----
+
+Usage: ``(nil? x)``
+
+Return True if x is nil/None.
+
+.. code-block:: clojure
+
+   => (nil? nil)
+   True
+
+   => (nil? None)
+   True
+
+   => (nil? 0)
+   False
+
+   => (setf x nil)
+   => (nil? x)
+   True
+
+   => ;; list.append always returns None
+   => (nil? (.append [1 2 3] 4))
+   True
+
+
 .. _none?-fn:
 
 none?
@@ -397,7 +427,7 @@ Return True if x is a string.
 .. _zero?-fn:
 
 zero?
-----
+-----
 
 Usage: ``(zero? x)``
 
@@ -575,6 +605,26 @@ See also :ref:`remove-fn`.
    => (list (filter even? [1 2 3 -4 5 -7]))
    [2, -4]
 
+.. _flatten-fn:
+
+flatten
+-------
+
+.. versionadded:: 0.9.12
+
+Usage: ``(flatten coll)``
+
+Return a single list of all the items in ``coll``, by flattening all
+contained lists and/or tuples.
+
+.. code-block:: clojure
+
+   => (flatten [1 2 [3 4] 5])
+   [1, 2, 3, 4, 5]
+
+   => (flatten ["foo" (, 1 2) [1 [2 3] 4] "bar"])
+   ['foo', 1, 2, 1, 2, 3, 4, 'bar']
+
 
 .. _iterate-fn:
 

docs/language/index.rst

@@ -11,3 +11,4 @@ Contents:
    api
    core
    internals
+   readermacros

docs/language/internals.rst

@@ -2,26 +2,278 @@
 Internal Hy Documentation
 =========================
 
-.. info::
+.. note::
-    These bits are for folks who hack on Hy it's self, mostly!
+    These bits are for folks who hack on Hy itself, mostly!
 
 
 Hy Models
 =========
 
-.. TODO::
+.. todo::
     Write this.
 
 
+Hy Internal Theory
+==================
+
+.. _overview:
+
+Overview
+--------
+
+The Hy internals work by acting as a front-end to Python bytecode, so that
+Hy it's self compiles down to Python Bytecode, allowing an unmodified Python
+runtime to run Hy.
+
+The way we do this is by translating Hy into Python AST, and building that AST
+down into Python bytecode using standard internals, so that we don't have
+to duplicate all the work of the Python internals for every single Python
+release.
+
+Hy works in four stages. The following sections will cover each step of Hy
+from source to runtime.
+
+.. _lexing:
+
+Lexing / tokenizing
+-------------------
+
+The first stage of compiling hy is to lex the source into tokens that we can
+deal with. We use a project called rply, which is a really nice (and fast)
+parser, written in a subset of Python called rpython.
+
+The lexing code is all defined in ``hy.lex.lexer``. This code is mostly just
+defining the Hy grammer, and all the actual hard parts are taken care of by
+rply -- we just define "callbacks" for rply in ``hy.lex.parser``, which take
+the tokens generated, and return the Hy models.
+
+You can think of the Hy models as the "AST" for Hy, it's what Macros operate
+on (directly), and it's what the compiler uses when it compiles Hy down.
+
+Check the documentation for more information on the Hy models for more
+information regarding the Hy models, and what they mean.
+
+.. TODO: Uh, we should, like, document models.
+
+
+.. _compiling:
+
+Compiling
+---------
+
+This is where most of the magic in Hy happens. This is where we take Hy AST
+(the models), and compile them into Python AST. A couple of funky things happen
+here to work past a few problems in AST, and working in the compiler is some
+of the most important work we do have.
+
+The compiler is a bit complex, so don't feel bad if you don't grok it on the
+first shot, it may take a bit of time to get right.
+
+The main entry-point to the Compiler is ``HyASTCompiler.compile``. This method
+is invoked, and the only real "public" method on the class (that is to say,
+we don't really promise the API beyond that method).
+
+In fact, even internally, we don't recurse directly hardly ever, we almost
+always force the Hy tree through ``compile``, and will often do this with
+sub-elements of an expression that we have. It's up to the Type-based dispatcher
+to properly dispatch sub-elements.
+
+All methods that preform a compilation are marked with the ``@builds()``
+decorator. You can either pass the class of the Hy model that it compiles,
+or you can use a string for expressions. I'll clear this up in a second.
+
+First stage type-dispatch
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Let's start in the ``compile`` method. The first thing we do is check the
+Type of the thing we're building. We look up to see if we have a method that
+can build the ``type()`` that we have, and dispatch to the method that can
+handle it. If we don't have any methods that can build that type, we raise
+an internal ``Exception``.
+
+For instance, if we have a ``HyString``, we have an almost 1-to-1 mapping of
+Hy AST to Python AST. The ``compile_string`` method takes the ``HyString``, and
+returns an ``ast.Str()`` that's populated with the correct line-numbers and
+content.
+
+Macro-expand
+~~~~~~~~~~~~
+
+If we get a ``HyExpression``, we'll attempt to see if this is a known
+Macro, and push to have it expanded by invoking ``hy.macros.macroexpand``, then
+push the result back into ``HyASTCompiler.compile``.
+
+Second stage expression-dispatch
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The only special case is the ``HyExpression``, since we need to create different
+AST depending on the special form in question. For instance, when we hit an
+``(if true true false)``, we need to generate a ``ast.If``, and properly
+compile the sub-nodes. This is where the ``@builds()`` with a String as an
+argument comes in.
+
+For the ``compile_expression`` (which is defined with an
+``@builds(HyExpression)``) will dispatch based on the string of the first
+argument. If, for some reason, the first argument is not a string, it will
+properly handle that case as well (most likely by raising an ``Exception``).
+
+If the String isn't known to Hy, it will default to create an ``ast.Call``,
+which will try to do a runtime call (in Python, something like ``foo()``).
+
+Issues hit with Python AST
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Python AST is great; it's what's enabled us to write such a powerful project
+on top of Python without having to fight Python too hard. Like anything, we've
+had our fair share of issues, and here's a short list of the common ones you
+might run into.
+
+*Python differentiates between Statements and Expressions*.
+
+This might not sound like a big deal -- in fact, to most Python programmers,
+this will shortly become a "Well, yeah" moment.
+
+In Python, doing something like:
+
+``print for x in range(10): pass``, because ``print`` prints expressions, and
+``for`` isn't an expression, it's a control flow statement. Things like
+``1 + 1`` are Expressions, as is ``lambda x: 1 + x``, but other language
+features, such as ``if``, ``for``, or ``while`` are statements.
+
+Since they have no "value" to Python, this makes working in Hy hard, since
+doing something like ``(print (if true true false))`` is not just common, it's
+expected.
+
+As a result, we auto-mangle things using a ``Result`` object, where we offer
+up any ``ast.stmt`` that need to get run, and a single ``ast.expr`` that can
+be used to get the value of whatever was just run. Hy does this by forcing
+assignment to things while running.
+
+As example, the Hy::
+
+    (print (if true true false))
+
+Will turn into::
+
+    if True:
+        _mangled_name_here = True
+    else:
+        _mangled_name_here = False
+
+    print _mangled_name_here
+
+
+OK, that was a bit of a lie, since we actually turn that statement
+into::
+
+    print True if True else False
+
+By forcing things into an ``ast.expr`` if we can, but the general idea holds.
+
+
+Runtime
+-------
+
+After we have a Python AST tree that's complete, we can try and compile it to
+Python bytecode by pushing it through ``eval``. From here on out, we're no
+longer in control, and Python is taking care of everything. This is why things
+like Python tracebacks, pdb and django apps work.
+
+
 Hy Macros
 =========
 
-.. TODO::
+.. _using-gensym:
-    Write this.
+
+Using gensym for safer macros
+------------------------------
+
+When writing macros, one must be careful to avoid capturing external variables
+or using variable names that might conflict with user code.
+
+We will use an example macro ``nif`` (see http://letoverlambda.com/index.cl/guest/chap3.html#sec_5
+for a more complete description.) ``nif`` is an example, something like a numeric ``if``,
+where based on the expression, one of the 3 forms is called depending on if the
+expression is positive, zero or negative.
+
+A first pass might be someting like:
+
+.. code-block:: clojure
+
+   (defmacro nif [expr pos-form zero-form neg-form]
+     `(let [[obscure-name ~expr]]
+       (cond [(pos? obscure-name) ~pos-form]
+             [(zero? obscure-name) ~zero-form]
+             [(neg? obscure-name) ~neg-form])))
+
+where ``obsure-name`` is an attempt to pick some variable name as not to
+conflict with other code. But of course, while well-intentioned,
+this is no guarantee.
+
+The method :ref:`gensym` is designed to generate a new, unique symbol for just
+such an occasion. A much better version of ``nif`` would be:
+
+.. code-block:: clojure
+
+   (defmacro nif [expr pos-form zero-form neg-form]
+     (let [[g (gensym)]]
+       `(let [[~g ~expr]]
+          (cond [(pos? ~g) ~pos-form]
+                [(zero? ~g) ~zero-form]
+                [(neg? ~g) ~neg-form]))))
+
+This is an easy case, since there is only one symbol. But if there is
+a need for several gensym's there is a second macro :ref:`with-gensyms` that
+basically expands to a series of ``let`` statements:
+
+.. code-block:: clojure
+
+   (with-gensyms [a b c]
+     ...)
+
+expands to:
+
+.. code-block:: clojure
+
+   (let [[a (gensym)
+         [b (gensym)
+         [c (gensym)]]
+     ...)
+
+so our re-written ``nif`` would look like:
+
+.. code-block:: clojure
+
+   (defmacro nif [expr pos-form zero-form neg-form]
+     (with-gensyms [g]
+       `(let [[~g ~expr]]
+          (cond [(pos? ~g) ~pos-form]
+                [(zero? ~g) ~zero-form]
+                [(neg? ~g) ~neg-form]))))
+
+Finally, though we can make a new macro that does all this for us. :ref:`defmacro/g!` 
+will take all symbols that begin with ``g!`` and automatically call ``gensym`` with the
+remainder of the symbol. So ``g!a`` would become ``(gensym "a")``.
+
+Our final version of ``nif``, built with ``defmacro/g!`` becomes:
+
+.. code-block:: clojure
+
+   (defmacro/g! nif [expr pos-form zero-form neg-form]
+     `(let [[~g!res ~expr]]
+        (cond [(pos? ~g!res) ~pos-form]
+              [(zero? ~g!res) ~zero-form]
+              [(neg? ~g!res) ~neg-form]))))
+
+
+
+Checking macro arguments and raising exceptions
+-----------------------------------------------
+
 
 
 Hy Compiler Builtins
 ====================
 
-.. TODO::
+.. todo::
     Write this.

docs/language/readermacros.rst

@@ -0,0 +1,61 @@
+.. _reader-macros:
+
+.. highlight:: clj
+
+=============
+Reader Macros
+=============
+
+Reader macros gives LISP the power to modify and alter syntax on the fly.
+You don't want polish notation? A reader macro can easily do just that. Want
+Clojure's way of having a regex? Reader macros can also do this easily.
+
+
+Syntax
+======
+
+::
+
+    => (defreader ^ [expr] (print expr))
+    => #^(1 2 3 4)
+    (1 2 3 4)
+    => #^"Hello"
+    "Hello"
+    => #^1+2+3+4+3+2
+    1+2+3+4+3+2
+
+
+Implementation
+==============
+
+Hy uses ``defreader`` to define the reader symbol, and ``#`` as the dispatch
+character. ``#`` expands into ``(dispatch_reader_macro ...)`` where the symbol
+and expression is quoted, and then passed along to the correct function::
+
+    => (defreader ^ ...)
+    => #^()
+    ;=> (dispatch_reader_macro '^ '())
+
+
+``defreader`` takes a single character as symbol name for the reader macro,
+anything longer will return an error. Implementation wise, ``defreader``
+expands into a lambda covered with a decorator, this decorater saves the
+lambda in a dict with its module name and symbol.
+
+::
+
+    => (defreader ^ [expr] (print expr))
+    ;=> (with_decorator (hy.macros.reader ^) (fn [expr] (print expr)))
+
+
+Anything passed along is quoted, thus given to the function defined.
+
+::
+
+    => #^"Hello"
+    "Hello"
+
+.. warning::
+   Because of a limitation in Hy's lexer and parser, reader macros can't
+   redefine defined syntax such as ``()[]{}``. This will most likely be
+   adressed in the future.

eg/python3/futures/hello-world.hy

@@ -7,5 +7,5 @@
 
 (with-as (ThreadPoolExecutor 10) executor
   (setv jobs (list-comp (.submit executor task-to-do) (x (range 0 10))))
-  (for (future (as-completed jobs))
+  (for [future (as-completed jobs)]
     (.result future)))

hy/cmdline.py

@@ -5,6 +5,7 @@
 # Copyright (c) 2013 Konrad Hinsen <konrad.hinsen@fastmail.net>
 # Copyright (c) 2013 Thom Neale <twneale@gmail.com>
 # Copyright (c) 2013 Will Kahn-Greene <willg@bluesock.org>
+# Copyright (c) 2013 Bob Tolbert <bob@tolbert.org>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -32,7 +33,7 @@ import sys
 import hy
 
 from hy.lex import LexException, PrematureEndOfInput, tokenize
-from hy.compiler import hy_compile
+from hy.compiler import hy_compile, HyTypeError
 from hy.importer import ast_compile, import_buffer_to_module
 from hy.completer import completion
 
@@ -79,12 +80,16 @@ class HyREPL(code.InteractiveConsole):
                                          filename=filename)
 
     def runsource(self, source, filename='<input>', symbol='single'):
+        global SIMPLE_TRACEBACKS
         try:
             tokens = tokenize(source)
         except PrematureEndOfInput:
             return True
-        except LexException:
+        except LexException as e:
-            self.showsyntaxerror(filename)
+            if e.source is None:
+                e.source = source
+                e.filename = filename
+            sys.stderr.write(str(e))
             return False
 
         try:
@@ -92,6 +97,15 @@ class HyREPL(code.InteractiveConsole):
             if self.spy:
                 print_python_code(_ast)
             code = ast_compile(_ast, filename, symbol)
+        except HyTypeError as e:
+            if e.source is None:
+                e.source = source
+                e.filename = filename
+            if SIMPLE_TRACEBACKS:
+                sys.stderr.write(str(e))
+            else:
+                self.showtraceback()
+            return False
         except Exception:
             self.showtraceback()
             return False
@@ -154,22 +168,34 @@ def ideas_macro():
 require("hy.cmdline", "__console__")
 require("hy.cmdline", "__main__")
 
+SIMPLE_TRACEBACKS = True
+
 
 def run_command(source):
     try:
         import_buffer_to_module("__main__", source)
-    except LexException as exc:
+    except (HyTypeError, LexException) as e:
-        # TODO: This would be better if we had line, col info.
+        if SIMPLE_TRACEBACKS:
-        print(source)
+            sys.stderr.write(str(e))
-        print(repr(exc))
+            return 1
-        return 1
+        raise
+    except Exception:
+        raise
     return 0
 
 
 def run_file(filename):
     from hy.importer import import_file_to_module
-    import_file_to_module("__main__", filename)
+    try:
-    return 0  # right?
+        import_file_to_module("__main__", filename)
+    except (HyTypeError, LexException) as e:
+        if SIMPLE_TRACEBACKS:
+            sys.stderr.write(str(e))
+            return 1
+        raise
+    except Exception:
+        raise
+    return 0
 
 
 def run_repl(hr=None, spy=False):
@@ -218,6 +244,9 @@ def cmdline_handler(scriptname, argv):
 
     parser.add_argument("-v", action="version", version=VERSION)
 
+    parser.add_argument("--show-tracebacks", action="store_true",
+                        help="show complete tracebacks for Hy exceptions")
+
     # this will contain the script/program name and any arguments for it.
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
@@ -228,6 +257,10 @@ def cmdline_handler(scriptname, argv):
 
     options = parser.parse_args(argv[1:])
 
+    if options.show_tracebacks:
+        global SIMPLE_TRACEBACKS
+        SIMPLE_TRACEBACKS = False
+
     # reset sys.argv like Python
     sys.argv = options.args or [""]
 

hy/compiler.py

@@ -4,6 +4,7 @@
 # Copyright (c) 2013 Julien Danjou <julien@danjou.info>
 # Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
 # Copyright (c) 2013 James King <james@agentultra.com>
+# Copyright (c) 2013 Bob Tolbert <bob@tolbert.org>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -23,8 +24,6 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-from hy.errors import HyError
-
 from hy.models.lambdalist import HyLambdaListKeyword
 from hy.models.expression import HyExpression
 from hy.models.keyword import HyKeyword
@@ -36,6 +35,8 @@ from hy.models.float import HyFloat
 from hy.models.list import HyList
 from hy.models.dict import HyDict
 
+from hy.errors import HyCompileError, HyTypeError
+
 import hy.macros
 from hy.macros import require, macroexpand
 from hy._compat import str_type, long_type
@@ -72,34 +73,6 @@ def load_stdlib():
             _stdlib[e] = module
 
 
-class HyCompileError(HyError):
-    def __init__(self, exception, traceback=None):
-        self.exception = exception
-        self.traceback = traceback
-
-    def __str__(self):
-        if isinstance(self.exception, HyTypeError):
-            return str(self.exception)
-        if self.traceback:
-            tb = "".join(traceback.format_tb(self.traceback)).strip()
-        else:
-            tb = "No traceback available. 😟"
-        return("Internal Compiler Bug 😱\n⤷ %s: %s\nCompilation traceback:\n%s"
-               % (self.exception.__class__.__name__,
-                  self.exception, tb))
-
-
-class HyTypeError(TypeError):
-    def __init__(self, expression, message):
-        super(HyTypeError, self).__init__(message)
-        self.expression = expression
-
-    def __str__(self):
-        return (super(HyTypeError, self).__str__() + " (line %s, column %d)"
-                % (self.expression.start_line,
-                   self.expression.start_column))
-
-
 _compile_table = {}
 
 
@@ -341,7 +314,7 @@ def checkargs(exact=None, min=None, max=None, even=None):
             if min is not None and (len(expression) - 1) < min:
                 _raise_wrong_args_number(
                     expression,
-                    "`%%s' needs at least %d arguments, got %%d" % (min))
+                    "`%%s' needs at least %d arguments, got %%d." % (min))
 
             if max is not None and (len(expression) - 1) > max:
                 _raise_wrong_args_number(
@@ -430,6 +403,8 @@ class HyASTCompiler(object):
             # nested; so let's re-raise this exception, let's not wrap it in
             # another HyCompileError!
             raise
+        except HyTypeError as e:
+            raise
         except Exception as e:
             raise HyCompileError(e, sys.exc_info()[2])
 
@@ -1086,18 +1061,25 @@ class HyASTCompiler(object):
         return rimports
 
     @builds("get")
-    @checkargs(2)
+    @checkargs(min=2)
     def compile_index_expression(self, expr):
         expr.pop(0)  # index
-        val = self.compile(expr.pop(0))  # target
-        sli = self.compile(expr.pop(0))  # slice
 
-        return val + sli + ast.Subscript(
+        val = self.compile(expr.pop(0))
-            lineno=expr.start_line,
+        slices, ret = self._compile_collect(expr)
-            col_offset=expr.start_column,
+
-            value=val.force_expr,
+        if val.stmts:
-            slice=ast.Index(value=sli.force_expr),
+            ret += val
-            ctx=ast.Load())
+
+        for sli in slices:
+            val = Result() + ast.Subscript(
+                lineno=expr.start_line,
+                col_offset=expr.start_column,
+                value=val.force_expr,
+                slice=ast.Index(value=sli),
+                ctx=ast.Load())
+
+        return ret + val
 
     @builds("del")
     @checkargs(min=1)
@@ -1177,14 +1159,18 @@ class HyASTCompiler(object):
         fn.stmts[-1].decorator_list = decorators
         return ret + fn
 
-    @builds("with")
+    @builds("with*")
     @checkargs(min=2)
     def compile_with_expression(self, expr):
-        expr.pop(0)  # with
+        expr.pop(0)  # with*
 
         args = expr.pop(0)
-        if len(args) > 2 or len(args) < 1:
+        if not isinstance(args, HyList):
-            raise HyTypeError(expr, "with needs [arg (expr)] or [(expr)]")
+            raise HyTypeError(expr,
+                              "with expects a list, received `{0}'".format(
+                                  type(args).__name__))
+        if len(args) < 1:
+            raise HyTypeError(expr, "with needs [[arg (expr)]] or [[(expr)]]]")
 
         args.reverse()
         ctx = self.compile(args.pop(0))
@@ -1573,6 +1559,9 @@ class HyASTCompiler(object):
                 fn.replace(ofn)
 
                 # Get the object we want to take an attribute from
+                if len(expression) < 2:
+                    raise HyTypeError(expression,
+                                      "attribute access requires object")
                 func = self.compile(expression.pop(1))
 
                 # And get the attribute
@@ -1623,23 +1612,36 @@ class HyASTCompiler(object):
         result += ld_name
         return result
 
-    @builds("foreach")
+    @builds("for*")
     @checkargs(min=1)
     def compile_for_expression(self, expression):
         expression.pop(0)  # for
-        target_name, iterable = expression.pop(0)
+
+        args = expression.pop(0)
+
+        if not isinstance(args, HyList):
+            raise HyTypeError(expression,
+                              "for expects a list, received `{0}'".format(
+                                  type(args).__name__))
+
+        try:
+            target_name, iterable = args
+        except ValueError:
+            raise HyTypeError(expression,
+                              "for requires two forms in the list")
+
         target = self._storeize(self.compile(target_name))
 
         ret = Result()
 
         orel = Result()
-        # (foreach [] body (else …))
+        # (for* [] body (else …))
         if expression and expression[-1][0] == HySymbol("else"):
             else_expr = expression.pop()
             if len(else_expr) > 2:
                 raise HyTypeError(
                     else_expr,
-                    "`else' statement in `foreach' is too long")
+                    "`else' statement in `for' is too long")
             elif len(else_expr) == 2:
                 orel += self.compile(else_expr[1])
                 orel += orel.expr_as_stmt()
@@ -1849,7 +1851,7 @@ class HyASTCompiler(object):
         expression.pop(0)
         name = expression.pop(0)
         NOT_READERS = [":", "&"]
-        if name in NOT_READERS:
+        if name in NOT_READERS or len(name) > 1:
             raise NameError("%s can't be used as a macro reader symbol" % name)
         if not isinstance(name, HySymbol):
             raise HyTypeError(name,

hy/contrib/anaphoric.hy

@@ -31,14 +31,14 @@
 
 (defmacro ap-each [lst &rest body]
   "Evaluate the body form for each element in the list."
-  `(foreach [it ~lst] ~@body))
+  `(for [it ~lst] ~@body))
 
 
 (defmacro ap-each-while [lst form &rest body]
   "Evalutate the body form for each element in the list while the
   predicate form evaluates to True."
   `(let [[p (lambda [it] ~form)]]
-     (foreach [it ~lst]
+     (for [it ~lst]
        (if (p it)
          ~@body
          (break)))))
@@ -47,7 +47,7 @@
 (defmacro ap-map [form lst]
   "Yield elements evaluated in the form for each element in the list."
   `(let [[f (lambda [it] ~form)]]
-     (foreach [v ~lst]
+     (for [v ~lst]
        (yield (f v)))))
 
 
@@ -55,7 +55,7 @@
   "Yield elements evaluated for each element in the list when the
   predicate function returns True."
   `(let [[f (lambda [it] ~rep)]]
-     (foreach [it ~lst]
+     (for [it ~lst]
        (if (~predfn it)
          (yield (f it))
          (yield it)))))
@@ -64,7 +64,7 @@
 (defmacro ap-filter [form lst]
   "Yield elements returned when the predicate form evaluates to True."
   `(let [[pred (lambda [it] ~form)]]
-     (foreach [val ~lst]
+     (for [val ~lst]
        (if (pred val)
          (yield val)))))
 

hy/core/bootstrap.hy

@@ -28,13 +28,13 @@
 
 (defmacro macro-error [location reason]
   "error out properly within a macro"
-  `(raise (hy.compiler.HyTypeError ~location ~reason)))
+  `(raise (hy.errors.HyMacroExpansionError ~location ~reason)))
 
 
 (defmacro defmacro-alias [names lambda-list &rest body]
   "define one macro with several names"
   (setv ret `(do))
-  (foreach [name names]
+  (for* [name names]
     (.append ret
              `(defmacro ~name ~lambda-list ~@body)))
   ret)
@@ -52,7 +52,7 @@
   (setv macroed_variables [])
   (if (not (isinstance variables HyList))
     (macro-error variables "let lexical context must be a list"))
-  (foreach [variable variables]
+  (for* [variable variables]
     (if (isinstance variable HyList)
       (do
        (if (!= (len variable) 2)

hy/core/language.hy

@@ -33,11 +33,11 @@
 (defn cycle [coll]
   "Yield an infinite repetition of the items in coll"
   (setv seen [])
-  (foreach [x coll]
+  (for* [x coll]
     (yield x)
     (.append seen x))
   (while seen
-    (foreach [x seen]
+    (for* [x seen]
       (yield x))))
 
 (defn dec [n]
@@ -49,7 +49,7 @@
   "Return a generator from the original collection with duplicates
    removed"
   (let [[seen []] [citer (iter coll)]]
-    (foreach [val citer]
+    (for* [val citer]
       (if (not_in val seen)
         (do
          (yield val)
@@ -58,7 +58,7 @@
 (defn drop [count coll]
   "Drop `count` elements from `coll` and yield back the rest"
   (let [[citer (iter coll)]]
-    (try (foreach [i (range count)]
+    (try (for* [i (range count)]
            (next citer))
          (catch [StopIteration]))
     citer))
@@ -66,10 +66,10 @@
 (defn drop-while [pred coll]
   "Drop all elements of `coll` until `pred` is False"
   (let [[citer (iter coll)]]
-    (foreach [val citer]
+    (for* [val citer]
       (if (not (pred val))
         (do (yield val) (break))))
-    (foreach [val citer]
+    (for* [val citer]
       (yield val))))
 
 (defn empty? [coll]
@@ -84,7 +84,7 @@
 (defn filter [pred coll]
   "Return all elements from `coll` that pass `pred`"
   (let [[citer (iter coll)]]
-    (foreach [val citer]
+    (for* [val citer]
       (if (pred val)
         (yield val)))))
 
@@ -96,7 +96,7 @@
 
 (defn _flatten [coll result]
   (if (and (iterable? coll) (not (string? coll)))
-    (do (foreach [b coll]
+    (do (for* [b coll]
           (_flatten b result)))
     (.append result coll))
   result)
@@ -160,6 +160,10 @@
   "Return true if x is None"
   (is x None))
 
+(defn nil? [x]
+  "Return true if x is nil (None)"
+  (is x None))
+
 (defn numeric? [x]
   (import numbers)
   (instance? numbers.Number x))
@@ -187,7 +191,7 @@
 (defn remove [pred coll]
   "Return coll with elements removed that pass `pred`"
   (let [[citer (iter coll)]]
-    (foreach [val citer]
+    (for* [val citer]
       (if (not (pred val))
         (yield val)))))
 
@@ -195,7 +199,7 @@
   "Yield x forever or optionally n times"
   (if (none? n)
     (setv dispatch (fn [] (while true (yield x))))
-    (setv dispatch (fn [] (foreach [_ (range n)] (yield x)))))
+    (setv dispatch (fn [] (for* [_ (range n)] (yield x)))))
   (dispatch))
 
 (defn repeatedly [func]
@@ -223,7 +227,7 @@
   "Take `count` elements from `coll`, or the whole set if the total
     number of entries in `coll` is less than `count`."
   (let [[citer (iter coll)]]
-    (foreach [_ (range count)]
+    (for* [_ (range count)]
       (yield (next citer)))))
 
 (defn take-nth [n coll]
@@ -231,16 +235,16 @@
      raises ValueError for (not (pos? n))"
   (if (pos? n)
     (let [[citer (iter coll)] [skip (dec n)]]
-      (foreach [val citer]
+      (for* [val citer]
         (yield val)
-        (foreach [_ (range skip)]
+        (for* [_ (range skip)]
           (next citer))))
     (raise (ValueError "n must be positive"))))
 
 (defn take-while [pred coll]
   "Take all elements while `pred` is true"
   (let [[citer (iter coll)]]
-    (foreach [val citer]
+    (for* [val citer]
       (if (pred val)
         (yield val)
         (break)))))
@@ -253,5 +257,5 @@
 (def *exports* '[cycle dec distinct drop drop-while empty? even? filter flatten
                  float? gensym
                  inc instance? integer integer? iterable? iterate iterator? neg?
-                 none? nth numeric? odd? pos? remove repeat repeatedly second
+                 nil? none? nth numeric? odd? pos? remove repeat repeatedly second
                  string string? take take-nth take-while zero?])

hy/core/macros.hy

@@ -25,21 +25,51 @@
 ;;; These macros form the hy language
 ;;; They are automatically required in every module, except inside hy.core
 
+
+(import [hy.models.list [HyList]]
+        [hy.models.symbol [HySymbol]])
+
+
+
 (defmacro for [args &rest body]
-  "shorthand for nested foreach loops:
+  "shorthand for nested for loops:
-  (for [x foo y bar] baz) ->
+  (for [x foo
-  (foreach [x foo]
+        y bar]
-    (foreach [y bar]
+    baz) ->
+  (for* [x foo]
+    (for* [y bar]
       baz))"
-  ;; TODO: that signature sucks.
+
-  ;; (for [[x foo] [y bar]] baz) would be more consistent
+  (if (odd? (len args))
-  (if (% (len args) 2)
+    (macro-error args "`for' requires an even number of args."))
-    (macro-error args "for needs an even number of elements in its first argument"))
+
+  (if (empty? body)
+    (macro-error None "`for' requires a body to evaluate"))
   (if args
-    `(foreach [~(.pop args 0) ~(.pop args 0)] (for ~args ~@body))
+    `(for* [~(.pop args 0) ~(.pop args 0)]
+       (for ~args ~@body))
     `(do ~@body)))
 
 
+(defmacro with [args &rest body]
+  "shorthand for nested for* loops:
+  (with [[x foo] [y bar]] baz) ->
+  (with* [x foo]
+    (with* [y bar]
+      baz))"
+
+  (if (not (empty? args))
+    (let [[primary (.pop args 0)]]
+      (if (isinstance primary HyList)
+        ;;; OK. if we have a list, we can go ahead and unpack that
+        ;;; as the argument to with.
+        `(with* [~@primary] (with ~args ~@body))
+        ;;; OK, let's just give it away. This may not be something we
+        ;;; can do, but that's really the programmer's problem.
+        `(with* [~primary] (with ~args ~@body))))
+      `(do ~@body)))
+
+
 (defmacro-alias [car first] [thing]
   "Get the first element of a list/cons"
   `(get ~thing 0))
@@ -71,7 +101,7 @@
   (setv root (check-branch branch))
   (setv latest-branch root)
 
-  (foreach [branch branches]
+  (for* [branch branches]
     (setv cur-branch (check-branch branch))
     (.append latest-branch cur-branch)
     (setv latest-branch cur-branch))
@@ -81,7 +111,7 @@
 (defmacro -> [head &rest rest]
   ;; TODO: fix the docstring by someone who understands this
   (setv ret head)
-  (foreach [node rest]
+  (for* [node rest]
     (if (not (isinstance node HyExpression))
       (setv node `(~node)))
     (.insert node 1 ret)
@@ -92,7 +122,7 @@
 (defmacro ->> [head &rest rest]
   ;; TODO: fix the docstring by someone who understands this
   (setv ret head)
-  (foreach [node rest]
+  (for* [node rest]
     (if (not (isinstance node HyExpression))
       (setv node `(~node)))
     (.append node ret)
@@ -113,7 +143,7 @@
 (defmacro yield-from [iterable]
   "Yield all the items from iterable"
   (let [[x (gensym)]]
-  `(foreach [~x ~iterable]
+  `(for* [~x ~iterable]
      (yield ~x))))
 
 (defmacro with-gensyms [args &rest body]

hy/errors.py

@@ -1,4 +1,7 @@
+# -*- encoding: utf-8 -*-
+#
 # Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
+# Copyright (c) 2013 Bob Tolbert <bob@tolbert.org>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -18,6 +21,8 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
+import traceback
+
 
 class HyError(Exception):
     """
@@ -25,3 +30,115 @@ class HyError(Exception):
     Exception.
     """
     pass
+
+
+try:
+    from clint.textui import colored
+except Exception:
+    class colored:
+
+        @staticmethod
+        def black(foo):
+            return foo
+
+        @staticmethod
+        def red(foo):
+            return foo
+
+        @staticmethod
+        def green(foo):
+            return foo
+
+        @staticmethod
+        def yellow(foo):
+            return foo
+
+        @staticmethod
+        def blue(foo):
+            return foo
+
+        @staticmethod
+        def magenta(foo):
+            return foo
+
+        @staticmethod
+        def cyan(foo):
+            return foo
+
+        @staticmethod
+        def white(foo):
+            return foo
+
+
+class HyCompileError(HyError):
+    def __init__(self, exception, traceback=None):
+        self.exception = exception
+        self.traceback = traceback
+
+    def __str__(self):
+        if isinstance(self.exception, HyTypeError):
+            return str(self.exception)
+        if self.traceback:
+            tb = "".join(traceback.format_tb(self.traceback)).strip()
+        else:
+            tb = "No traceback available. 😟"
+        return("Internal Compiler Bug 😱\n⤷ %s: %s\nCompilation traceback:\n%s"
+               % (self.exception.__class__.__name__,
+                  self.exception, tb))
+
+
+class HyTypeError(TypeError):
+    def __init__(self, expression, message):
+        super(HyTypeError, self).__init__(message)
+        self.expression = expression
+        self.message = message
+        self.source = None
+        self.filename = None
+
+    def __str__(self):
+
+        line = self.expression.start_line
+        start = self.expression.start_column
+        end = self.expression.end_column
+
+        source = []
+        if self.source is not None:
+            source = self.source.split("\n")[line-1:self.expression.end_line]
+
+            if line == self.expression.end_line:
+                length = end - start
+            else:
+                length = len(source[0]) - start
+
+        result = ""
+
+        result += '  File "%s", line %d, column %d\n\n' % (self.filename,
+                                                           line,
+                                                           start)
+
+        if len(source) == 1:
+            result += '  %s\n' % colored.red(source[0])
+            result += '  %s%s\n' % (' '*(start-1),
+                                    colored.green('^' + '-'*(length-1) + '^'))
+        if len(source) > 1:
+            result += '  %s\n' % colored.red(source[0])
+            result += '  %s%s\n' % (' '*(start-1),
+                                    colored.green('^' + '-'*length))
+            if len(source) > 2:  # write the middle lines
+                for line in source[1:-1]:
+                    result += '  %s\n' % colored.red("".join(line))
+                    result += '  %s\n' % colored.green("-"*len(line))
+
+            # write the last line
+            result += '  %s\n' % colored.red("".join(source[-1]))
+            result += '  %s\n' % colored.green('-'*(end-1) + '^')
+
+        result += colored.yellow("%s: %s\n\n" %
+                                 (self.__class__.__name__,
+                                  self.message))
+
+        return result
+
+
+class HyMacroExpansionError(HyTypeError):
+    pass

hy/importer.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
+# Copyright (c) 2013 Bob Tolbert <bob@tolbert.org>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -18,9 +19,9 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-from hy.compiler import hy_compile
+from hy.compiler import hy_compile, HyTypeError
 from hy.models import HyObject
-from hy.lex import tokenize
+from hy.lex import tokenize, LexException
 
 from io import open
 import marshal
@@ -71,17 +72,28 @@ def import_file_to_module(module_name, fpath):
         mod = imp.new_module(module_name)
         mod.__file__ = fpath
         eval(ast_compile(_ast, fpath, "exec"), mod.__dict__)
+    except (HyTypeError, LexException) as e:
+        if e.source is None:
+            with open(fpath, 'rt') as fp:
+                e.source = fp.read()
+            e.filename = fpath
+        raise
     except Exception:
         sys.modules.pop(module_name, None)
         raise
-
     return mod
 
 
 def import_buffer_to_module(module_name, buf):
-    _ast = import_buffer_to_ast(buf, module_name)
+    try:
-    mod = imp.new_module(module_name)
+        _ast = import_buffer_to_ast(buf, module_name)
-    eval(ast_compile(_ast, "", "exec"), mod.__dict__)
+        mod = imp.new_module(module_name)
+        eval(ast_compile(_ast, "", "exec"), mod.__dict__)
+    except (HyTypeError, LexException) as e:
+        if e.source is None:
+            e.source = buf
+            e.filename = '<stdin>'
+        raise
     return mod
 
 

hy/lex/__init__.py

@@ -33,6 +33,5 @@ def tokenize(buf):
         return parser.parse(lexer.lex(buf))
     except LexingError as e:
         pos = e.getsourcepos()
-        raise LexException(
+        raise LexException("Could not identify the next token.",
-            "Could not identify the next token at line %s, column %s" % (
+                           pos.lineno, pos.colno)
-                pos.lineno, pos.colno))

hy/lex/exceptions.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
+# Copyright (c) 2013 Bob Tolbert <bob@tolbert.org>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -23,9 +24,43 @@ from hy.errors import HyError
 
 class LexException(HyError):
     """Error during the Lexing of a Hython expression."""
-    pass
+    def __init__(self, message, lineno, colno):
+        super(LexException, self).__init__(message)
+        self.message = message
+        self.lineno = lineno
+        self.colno = colno
+        self.source = None
+        self.filename = '<stdin>'
+
+    def __str__(self):
+        from hy.errors import colored
+
+        line = self.lineno
+        start = self.colno
+
+        result = ""
+
+        source = self.source.split("\n")
+
+        if line > 0 and start > 0:
+            result += '  File "%s", line %d, column %d\n\n' % (self.filename,
+                                                               line,
+                                                               start)
+
+            if len(self.source) > 0:
+                source_line = source[line-1]
+            else:
+                source_line = ""
+
+            result += '  %s\n' % colored.red(source_line)
+            result += '  %s%s\n' % (' '*(start-1), colored.green('^'))
+
+        result += colored.yellow("LexException: %s\n\n" % self.message)
+
+        return result
 
 
 class PrematureEndOfInput(LexException):
     """We got a premature end of input"""
-    pass
+    def __init__(self, message):
+        super(PrematureEndOfInput, self).__init__(message, -1, -1)

hy/lex/parser.py

@@ -229,6 +229,7 @@ def t_identifier(p):
     table = {
         "true": "True",
         "false": "False",
+        "nil": "None",
         "null": "None",
     }
 
@@ -257,12 +258,11 @@ def t_identifier(p):
 def error_handler(token):
     tokentype = token.gettokentype()
     if tokentype == '$end':
-        raise PrematureEndOfInput
+        raise PrematureEndOfInput("Premature end of input")
     else:
         raise LexException(
-            "Ran into a %s where it wasn't expected at line %s, column %s" %
+            "Ran into a %s where it wasn't expected." % tokentype,
-            (tokentype, token.source_pos.lineno, token.source_pos.colno)
+            token.source_pos.lineno, token.source_pos.colno)
-        )
 
 
 parser = pg.build()

hy/macros.py

@@ -28,6 +28,8 @@ from hy.models.complex import HyComplex
 from hy.models.dict import HyDict
 from hy._compat import str_type, long_type
 
+from hy.errors import HyTypeError, HyMacroExpansionError
+
 from collections import defaultdict
 import sys
 
@@ -192,7 +194,16 @@ def macroexpand_1(tree, module_name):
             if m is None:
                 m = _hy_macros[None].get(fn)
             if m is not None:
-                obj = _wrap_value(m(*ntree[1:]))
+                try:
+                    obj = _wrap_value(m(*ntree[1:]))
+                except HyTypeError as e:
+                    if e.expression is None:
+                        e.expression = tree
+                    raise
+                except Exception as e:
+                    msg = "`" + str(tree[0]) + "' " + \
+                          " ".join(str(e).split()[1:])
+                    raise HyMacroExpansionError(tree, msg)
                 obj.replace(tree)
                 return obj
 

hy/version.py

@@ -20,4 +20,4 @@
 
 
 __appname__ = "hy"
-__version__ = "0.9.11"
+__version__ = "0.9.12"

scripts/update_coreteam.py

@@ -19,7 +19,6 @@ MISSING_NAMES = {
 # an owner of the organization.
 CONCEALED_MEMBERS = [
     ('aldeka', 'Karen Rustad'),
-    ('rwtolbert', 'Bob Tolbert'),
     ('tuturto', 'Tuukka Turto'),
 ]
 

tests/__init__.py

@@ -12,4 +12,5 @@ from .native_tests.when import *  # noqa
 from .native_tests.with_decorator import *  # noqa
 from .native_tests.core import *  # noqa
 from .native_tests.reader_macros import *  # noqa
+from .native_tests.with_test import *  # noqa
 from .native_tests.contrib.anaphoric import *  # noqa

tests/compilers/test_ast.py

@@ -22,7 +22,10 @@
 from __future__ import unicode_literals
 
 from hy import HyString
-from hy.compiler import hy_compile, HyCompileError, HyTypeError
+from hy.models import HyObject
+from hy.compiler import hy_compile
+from hy.errors import HyCompileError, HyTypeError
+from hy.lex.exceptions import LexException
 from hy.lex import tokenize
 
 import ast
@@ -42,10 +45,14 @@ def can_compile(expr):
 
 
 def cant_compile(expr):
-    expr = tokenize(expr)
     try:
-        hy_compile(expr, "__main__")
+        hy_compile(tokenize(expr), "__main__")
         assert False
+    except HyTypeError as e:
+        # Anything that can't be compiled should raise a user friendly
+        # error, otherwise it's a compiler bug.
+        assert isinstance(e.expression, HyObject)
+        assert e.message
     except HyCompileError as e:
         # Anything that can't be compiled should raise a user friendly
         # error, otherwise it's a compiler bug.
@@ -253,7 +260,6 @@ def test_ast_bad_get():
     "Make sure AST can't compile invalid get"
     cant_compile("(get)")
     cant_compile("(get 1)")
-    cant_compile("(get 1 2 3)")
 
 
 def test_ast_good_slice():
@@ -295,9 +301,9 @@ def test_ast_bad_assoc():
 
 def test_ast_bad_with():
     "Make sure AST can't compile invalid with"
-    cant_compile("(with)")
+    cant_compile("(with*)")
-    cant_compile("(with [])")
+    cant_compile("(with* [])")
-    cant_compile("(with [] (pass))")
+    cant_compile("(with* [] (pass))")
 
 
 def test_ast_valid_while():
@@ -305,14 +311,14 @@ def test_ast_valid_while():
     can_compile("(while foo bar)")
 
 
-def test_ast_valid_foreach():
+def test_ast_valid_for():
-    "Make sure AST can compile valid foreach"
+    "Make sure AST can compile valid for"
-    can_compile("(foreach [a 2])")
+    can_compile("(for [a 2] (print a))")
 
 
-def test_ast_invalid_foreach():
+def test_ast_invalid_for():
-    "Make sure AST can't compile invalid foreach"
+    "Make sure AST can't compile invalid for"
-    cant_compile("(foreach [a 1] (else 1 2))")
+    cant_compile("(for* [a 1] (else 1 2))")
 
 
 def test_ast_expression_basics():
@@ -423,8 +429,38 @@ def test_compile_error():
     """Ensure we get compile error in tricky cases"""
     try:
         can_compile("(fn [] (= 1))")
-    except HyCompileError as e:
+    except HyTypeError as e:
-        assert(str(e)
+        assert(e.message == "`=' needs at least 2 arguments, got 1.")
-               == "`=' needs at least 2 arguments, got 1 (line 1, column 8)")
+    else:
+        assert(False)
+
+
+def test_for_compile_error():
+    """Ensure we get compile error in tricky 'for' cases"""
+    try:
+        can_compile("(fn [] (for)")
+    except LexException as e:
+        assert(e.message == "Premature end of input")
+    else:
+        assert(False)
+
+    try:
+        can_compile("(fn [] (for)))")
+    except LexException as e:
+        assert(e.message == "Ran into a RPAREN where it wasn't expected.")
+    else:
+        assert(False)
+
+    try:
+        can_compile("(fn [] (for [x]))")
+    except HyTypeError as e:
+        assert(e.message == "`for' requires an even number of args.")
+    else:
+        assert(False)
+
+    try:
+        can_compile("(fn [] (for [x xx]))")
+    except HyTypeError as e:
+        assert(e.message == "`for' requires a body to evaluate")
     else:
         assert(False)

tests/native_tests/core.hy

@@ -293,6 +293,15 @@
   (assert-false (none? 0))
   (assert-false (none? "")))
 
+(defn test-nil? []
+  "NATIVE: testing for `is nil`"
+  (assert-true (nil? nil))
+  (assert-true (nil? None))
+  (setv f nil)
+  (assert-true (nil? f))
+  (assert-false (nil? 0))
+  (assert-false (nil? "")))
+
 (defn test-nth []
   "NATIVE: testing the nth function"
   (assert-equal 2 (nth [1 2 4 7] 1))

tests/native_tests/language.hy

@@ -86,9 +86,10 @@
 
 (defn test-is []
   "NATIVE: test is can deal with None"
-  (setv a null)
+  (setv a nil)
-  (assert (is a null))
+  (assert (is a nil))
-  (assert (is-not a "b")))
+  (assert (is-not a "b"))
+  (assert (none? a)))
 
 
 (defn test-branching []
@@ -129,7 +130,16 @@
 (defn test-index []
   "NATIVE: Test that dict access works"
   (assert (= (get {"one" "two"} "one") "two"))
-  (assert (= (get [1 2 3 4 5] 1) 2)))
+  (assert (= (get [1 2 3 4 5] 1) 2))
+  (assert (= (get {"first" {"second" {"third" "level"}}}
+                  "first" "second" "third")
+             "level"))
+  (assert (= (get ((fn [] {"first" {"second" {"third" "level"}}}))
+                  "first" "second" "third")
+             "level"))
+  (assert (= (get {"first" {"second" {"third" "level"}}}
+                  ((fn [] "first")) "second" "third")
+             "level")))
 
 
 (defn test-lambda []
@@ -439,37 +449,37 @@
 
 (defn test-context []
   "NATIVE: test with"
-  (with [fd (open "README.md" "r")] (assert fd))
+  (with [[fd (open "README.md" "r")]] (assert fd))
-  (with [(open "README.md" "r")] (do)))
+  (with [[(open "README.md" "r")]] (do)))
 
 
 (defn test-with-return []
   "NATIVE: test that with returns stuff"
   (defn read-file [filename]
-    (with [fd (open filename "r")] (.read fd)))
+    (with [[fd (open filename "r")]] (.read fd)))
   (assert (!= 0 (len (read-file "README.md")))))
 
 
 (defn test-for-doodle []
   "NATIVE: test for-do"
   (do (do (do (do (do (do (do (do (do (setv (, x y) (, 0 0)))))))))))
-  (foreach [- [1 2]]
+  (for [- [1 2]]
     (do
      (setv x (+ x 1))
      (setv y (+ y 1))))
   (assert (= y x 2)))
 
 
-(defn test-foreach-else []
+(defn test-for-else []
-  "NATIVE: test foreach else"
+  "NATIVE: test for else"
   (let [[x 0]]
-    (foreach [a [1 2]]
+    (for* [a [1 2]]
       (setv x (+ x a))
       (else (setv x (+ x 50))))
     (assert (= x 53)))
 
   (let [[x 0]]
-    (foreach [a [1 2]]
+    (for* [a [1 2]]
       (setv x (+ x a))
       (else))
     (assert (= x 3))))
@@ -807,9 +817,9 @@
 (defn test-continue-continuation []
   "NATIVE: test checking if continue actually continues"
   (setv y [])
-  (for [x (range 10)]
+  (for [x (range 10)] 
-    (if (!= x 5)
+    (if (!= x 5) 
-      (continue))
+      (continue)) 
     (.append y x))
   (assert (= y [5])))
 

tests/native_tests/native_macros.hy

@@ -56,7 +56,7 @@
 (defn test-midtree-yield-in-for []
   "NATIVE: test yielding in a for with a return"
   (defn kruft-in-for []
-    (for [i (range 5)]
+    (for* [i (range 5)]
       (yield i))
     (+ 1 2)))
 
@@ -72,7 +72,7 @@
 (defn test-multi-yield []
   "NATIVE: testing multiple yields"
   (defn multi-yield []
-    (for [i (range 3)]
+    (for* [i (range 3)]
       (yield i))
     (yield "a")
     (yield "end"))
@@ -97,7 +97,7 @@
 (defn test-yield-from []
   "NATIVE: testing yield from"
   (defn yield-from-test []
-    (for [i (range 3)]
+    (for* [i (range 3)]
       (yield i))
     (yield-from [1 2 3]))
   (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))

tests/native_tests/unless.hy

@@ -4,5 +4,7 @@
   (assert (= (unless false 1 2) 2))
   (assert (= (unless false 1 3) 3))
   (assert (= (unless true 2) null))
+  (assert (= (unless true 2) nil))
   (assert (= (unless (!= 1 2) 42) null))
+  (assert (= (unless (!= 1 2) 42) nil))
   (assert (= (unless (!= 2 2) 42) 42)))

tests/native_tests/when.hy

@@ -5,4 +5,6 @@
   (assert (= (when true 1 3) 3))
   (assert (= (when false 2) null))
   (assert (= (when (= 1 2) 42) null))
+  (assert (= (when false 2) nil))
+  (assert (= (when (= 1 2) 42) nil))
   (assert (= (when (= 2 2) 42) 42)))

tests/native_tests/with_test.hy

@@ -0,0 +1,44 @@
+(defclass WithTest [object]
+  [(--init-- 
+    (fn [self val]
+      (setv self.val val)
+      None))
+
+  (--enter--
+    (fn [self]
+      self.val))
+
+  (--exit--
+    (fn [self type value traceback]
+      (setv self.val None)))])
+
+(defn test-single-with []
+  "NATIVE: test a single with"
+  (with [[t (WithTest 1)]]
+    (assert (= t 1))))
+
+(defn test-two-with []
+  "NATIVE: test two withs"
+  (with [[t1 (WithTest 1)]
+         [t2 (WithTest 2)]]
+   (assert (= t1 1))
+   (assert (= t2 2))))
+
+(defn test-thrice-with []
+  "NATIVE: test three withs"
+  (with [[t1 (WithTest 1)]
+         [t2 (WithTest 2)]
+         [t3 (WithTest 3)]]
+    (assert (= t1 1))
+    (assert (= t2 2))
+    (assert (= t3 3))))
+
+(defn test-quince-with []
+  "NATIVE: test four withs, one with no args"
+    (with [[t1 (WithTest 1)]
+          [t2 (WithTest 2)]
+          [t3 (WithTest 3)]
+          [(WithTest 4)]]
+    (assert (= t1 1))
+    (assert (= t2 2))
+    (assert (= t3 3))))

tests/test_bin.py

@@ -64,7 +64,7 @@ def test_bin_hy_cmd():
 
     ret = run_cmd("hy -c \"(koan\"")
     assert ret[0] == 1
-    assert "PrematureEndOfInput" in ret[1]
+    assert "Premature end of input" in ret[2]
 
 
 def test_bin_hy_icmd():