From 55a7ab1667583d95a514e33a5df62c376078c2f5 Mon Sep 17 00:00:00 2001 From: Bob Tolbert Date: Fri, 27 Dec 2013 13:50:19 -0700 Subject: [PATCH 1/9] add nil as synonym for None --- hy/lex/parser.py | 1 + tests/native_tests/language.hy | 7 ++++--- tests/native_tests/unless.hy | 2 ++ tests/native_tests/when.hy | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hy/lex/parser.py b/hy/lex/parser.py index 70d43f8..9e11069 100644 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -229,6 +229,7 @@ def t_identifier(p): table = { "true": "True", "false": "False", + "nil": "None", "null": "None", } diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 4efe768..ec110ae 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -86,9 +86,10 @@ (defn test-is [] "NATIVE: test is can deal with None" - (setv a null) - (assert (is a null)) - (assert (is-not a "b"))) + (setv a nil) + (assert (is a nil)) + (assert (is-not a "b")) + (assert (none? a))) (defn test-branching [] diff --git a/tests/native_tests/unless.hy b/tests/native_tests/unless.hy index f8c298a..2b957fe 100644 --- a/tests/native_tests/unless.hy +++ b/tests/native_tests/unless.hy @@ -4,5 +4,7 @@ (assert (= (unless false 1 2) 2)) (assert (= (unless false 1 3) 3)) (assert (= (unless true 2) null)) + (assert (= (unless true 2) nil)) (assert (= (unless (!= 1 2) 42) null)) + (assert (= (unless (!= 1 2) 42) nil)) (assert (= (unless (!= 2 2) 42) 42))) diff --git a/tests/native_tests/when.hy b/tests/native_tests/when.hy index 1409b3b..c281439 100644 --- a/tests/native_tests/when.hy +++ b/tests/native_tests/when.hy @@ -5,4 +5,6 @@ (assert (= (when true 1 3) 3)) (assert (= (when false 2) null)) (assert (= (when (= 1 2) 42) null)) + (assert (= (when false 2) nil)) + (assert (= (when (= 1 2) 42) nil)) (assert (= (when (= 2 2) 42) 42))) From d82636958b010d5cdf34af22576f04ecb4ffd098 Mon Sep 17 00:00:00 2001 From: Foxboron Date: Sun, 10 Nov 2013 19:00:01 +0100 Subject: [PATCH 2/9] added for and with macros Fixed up anaphoric.hy and added the tests too native_tests/__init__.py added __init__.hy --- docs/language/api.rst | 52 +++++++++++------------------ hy/compiler.py | 33 +++++++++++++----- hy/contrib/anaphoric.hy | 10 +++--- hy/core/bootstrap.hy | 4 +-- hy/core/language.hy | 28 ++++++++-------- hy/core/macros.hy | 33 ++++++++++-------- tests/__init__.py | 1 + tests/compilers/test_ast.py | 18 +++++----- tests/native_tests/language.hy | 36 ++++++++++---------- tests/native_tests/native_macros.hy | 6 ++-- tests/native_tests/with_test.hy | 44 ++++++++++++++++++++++++ 11 files changed, 160 insertions(+), 105 deletions(-) create mode 100644 tests/native_tests/with_test.hy diff --git a/docs/language/api.rst b/docs/language/api.rst index 9934b4d..60f4d89 100644 --- a/docs/language/api.rst +++ b/docs/language/api.rst @@ -441,52 +441,36 @@ first / car for ---- - -`for` macro is used to build nested `foreach` loops. The macro takes two -parameters, first being a vector specifying collections to iterate over and -variables to bind. The second parameter is a statement which is executed during -each loop: - -.. code-block:: clj - - (for [x iter y iter] stmt) - - (foreach [x iter] - (foreach [y iter] stmt)) - - -foreach ------- -`foreach` is used to call a function for each element in a list or vector. +`for` is used to call a function for each element in a list or vector. Results are discarded and None is returned instead. Example code iterates over collection and calls side-effect to each element in the collection: .. code-block:: clj ;; assuming that (side-effect) is a function that takes a single parameter - (foreach [element collection] (side-effect element)) + (for [[element collection]] (side-effect element)) - ;; foreach can have an optional else block - (foreach [element collection] (side-effect element) - (else (side-effect-2))) + ;; for can have an optional else block + (for [[element collection]] (side-effect element) + (else (side-effect-2))) -The optional `else` block is executed only if the `foreach` loop terminates +The optional `else` block is executed only if the `for` loop terminates normally. If the execution is halted with `break`, the `else` does not execute. .. code-block:: clj - => (foreach [element [1 2 3]] (if (< element 3) - ... (print element) - ... (break)) + => (for [[element [1 2 3]]] (if (< element 3) + ... (print element) + ... (break)) ... (else (print "loop finished"))) 1 2 - => (foreach [element [1 2 3]] (if (< element 4) - ... (print element) - ... (break)) + => (for [[element [1 2 3]]] (if (< element 4) + ... (print element) + ... (break)) ... (else (print "loop finished"))) 1 2 @@ -635,7 +619,7 @@ function is defined and passed to another function for filtering output. ... {:name "Dave" :age 5}]) => (defn display-people [people filter] - ... (foreach [person people] (if (filter person) (print (:name person))))) + ... (for [[person people]] (if (filter person) (print (:name person))))) => (display-people people (fn [person] (< (:age person) 25))) Alice @@ -949,16 +933,18 @@ context to an argument or ignore it completely, as shown below: .. code-block:: clj - (with [arg (expr)] block) + (with [[arg (expr)]] block) - (with [(expr)] block) + (with [[(expr)]] block) + + (with [[arg (expr)] [(expr)]] block) The following example will open file `NEWS` and print its content on screen. The file is automatically closed after it has been processed. .. code-block:: clj - (with [f (open "NEWS")] (print (.read f))) + (with [[f (open "NEWS")]] (print (.read f))) with-decorator @@ -996,7 +982,7 @@ infinite series without consuming infinite amount of memory. .. code-block:: clj => (defn multiply [bases coefficients] - ... (foreach [(, base coefficient) (zip bases coefficients)] + ... (for [[(, base coefficient) (zip bases coefficients)]] ... (yield (* base coefficient)))) => (multiply (range 5) (range 5)) diff --git a/hy/compiler.py b/hy/compiler.py index 9df4fb9..9a3b710 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1177,14 +1177,18 @@ class HyASTCompiler(object): fn.stmts[-1].decorator_list = decorators return ret + fn - @builds("with") + @builds("with*") @checkargs(min=2) def compile_with_expression(self, expr): - expr.pop(0) # with + expr.pop(0) # with* args = expr.pop(0) - if len(args) > 2 or len(args) < 1: - raise HyTypeError(expr, "with needs [arg (expr)] or [(expr)]") + if not isinstance(args, HyList): + raise HyTypeError(expr, + "with expects a list, received `{0}'".format( + type(args).__name__)) + if len(args) < 1: + raise HyTypeError(expr, "with needs [[arg (expr)]] or [[(expr)]]]") args.reverse() ctx = self.compile(args.pop(0)) @@ -1623,23 +1627,36 @@ class HyASTCompiler(object): result += ld_name return result - @builds("foreach") + @builds("for*") @checkargs(min=1) def compile_for_expression(self, expression): expression.pop(0) # for - target_name, iterable = expression.pop(0) + + args = expression.pop(0) + + if not isinstance(args, HyList): + raise HyTypeError(expression, + "for expects a list, received `{0}'".format( + type(args).__name__)) + + try: + target_name, iterable = args + except ValueError: + raise HyTypeError(expression, + "for requires two forms in the list") + target = self._storeize(self.compile(target_name)) ret = Result() orel = Result() - # (foreach [] body (else …)) + # (for* [] body (else …)) if expression and expression[-1][0] == HySymbol("else"): else_expr = expression.pop() if len(else_expr) > 2: raise HyTypeError( else_expr, - "`else' statement in `foreach' is too long") + "`else' statement in `for' is too long") elif len(else_expr) == 2: orel += self.compile(else_expr[1]) orel += orel.expr_as_stmt() diff --git a/hy/contrib/anaphoric.hy b/hy/contrib/anaphoric.hy index c451559..9a281d0 100644 --- a/hy/contrib/anaphoric.hy +++ b/hy/contrib/anaphoric.hy @@ -31,14 +31,14 @@ (defmacro ap-each [lst &rest body] "Evaluate the body form for each element in the list." - `(foreach [it ~lst] ~@body)) + `(for [[it ~lst]] ~@body)) (defmacro ap-each-while [lst form &rest body] "Evalutate the body form for each element in the list while the predicate form evaluates to True." `(let [[p (lambda [it] ~form)]] - (foreach [it ~lst] + (for [[it ~lst]] (if (p it) ~@body (break))))) @@ -47,7 +47,7 @@ (defmacro ap-map [form lst] "Yield elements evaluated in the form for each element in the list." `(let [[f (lambda [it] ~form)]] - (foreach [v ~lst] + (for [[v ~lst]] (yield (f v))))) @@ -55,7 +55,7 @@ "Yield elements evaluated for each element in the list when the predicate function returns True." `(let [[f (lambda [it] ~rep)]] - (foreach [it ~lst] + (for [[it ~lst]] (if (~predfn it) (yield (f it)) (yield it))))) @@ -64,7 +64,7 @@ (defmacro ap-filter [form lst] "Yield elements returned when the predicate form evaluates to True." `(let [[pred (lambda [it] ~form)]] - (foreach [val ~lst] + (for [[val ~lst]] (if (pred val) (yield val))))) diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy index 007d92b..5d3d367 100644 --- a/hy/core/bootstrap.hy +++ b/hy/core/bootstrap.hy @@ -34,7 +34,7 @@ (defmacro defmacro-alias [names lambda-list &rest body] "define one macro with several names" (setv ret `(do)) - (foreach [name names] + (for* [name names] (.append ret `(defmacro ~name ~lambda-list ~@body))) ret) @@ -52,7 +52,7 @@ (setv macroed_variables []) (if (not (isinstance variables HyList)) (macro-error variables "let lexical context must be a list")) - (foreach [variable variables] + (for* [variable variables] (if (isinstance variable HyList) (do (if (!= (len variable) 2) diff --git a/hy/core/language.hy b/hy/core/language.hy index d03db93..9ac2fee 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -33,11 +33,11 @@ (defn cycle [coll] "Yield an infinite repetition of the items in coll" (setv seen []) - (foreach [x coll] + (for* [x coll] (yield x) (.append seen x)) (while seen - (foreach [x seen] + (for* [x seen] (yield x)))) (defn dec [n] @@ -49,7 +49,7 @@ "Return a generator from the original collection with duplicates removed" (let [[seen []] [citer (iter coll)]] - (foreach [val citer] + (for* [val citer] (if (not_in val seen) (do (yield val) @@ -58,7 +58,7 @@ (defn drop [count coll] "Drop `count` elements from `coll` and yield back the rest" (let [[citer (iter coll)]] - (try (foreach [i (range count)] + (try (for* [i (range count)] (next citer)) (catch [StopIteration])) citer)) @@ -66,10 +66,10 @@ (defn drop-while [pred coll] "Drop all elements of `coll` until `pred` is False" (let [[citer (iter coll)]] - (foreach [val citer] + (for* [val citer] (if (not (pred val)) (do (yield val) (break)))) - (foreach [val citer] + (for* [val citer] (yield val)))) (defn empty? [coll] @@ -84,7 +84,7 @@ (defn filter [pred coll] "Return all elements from `coll` that pass `pred`" (let [[citer (iter coll)]] - (foreach [val citer] + (for* [val citer] (if (pred val) (yield val))))) @@ -96,7 +96,7 @@ (defn _flatten [coll result] (if (and (iterable? coll) (not (string? coll))) - (do (foreach [b coll] + (do (for* [b coll] (_flatten b result))) (.append result coll)) result) @@ -187,7 +187,7 @@ (defn remove [pred coll] "Return coll with elements removed that pass `pred`" (let [[citer (iter coll)]] - (foreach [val citer] + (for* [val citer] (if (not (pred val)) (yield val))))) @@ -195,7 +195,7 @@ "Yield x forever or optionally n times" (if (none? n) (setv dispatch (fn [] (while true (yield x)))) - (setv dispatch (fn [] (foreach [_ (range n)] (yield x))))) + (setv dispatch (fn [] (for* [_ (range n)] (yield x))))) (dispatch)) (defn repeatedly [func] @@ -223,7 +223,7 @@ "Take `count` elements from `coll`, or the whole set if the total number of entries in `coll` is less than `count`." (let [[citer (iter coll)]] - (foreach [_ (range count)] + (for* [_ (range count)] (yield (next citer))))) (defn take-nth [n coll] @@ -231,16 +231,16 @@ raises ValueError for (not (pos? n))" (if (pos? n) (let [[citer (iter coll)] [skip (dec n)]] - (foreach [val citer] + (for* [val citer] (yield val) - (foreach [_ (range skip)] + (for* [_ (range skip)] (next citer)))) (raise (ValueError "n must be positive")))) (defn take-while [pred coll] "Take all elements while `pred` is true" (let [[citer (iter coll)]] - (foreach [val citer] + (for* [val citer] (if (pred val) (yield val) (break))))) diff --git a/hy/core/macros.hy b/hy/core/macros.hy index d7c8b15..39338c1 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -26,17 +26,24 @@ ;;; They are automatically required in every module, except inside hy.core (defmacro for [args &rest body] - "shorthand for nested foreach loops: - (for [x foo y bar] baz) -> - (foreach [x foo] - (foreach [y bar] + "shorthand for nested for loops: + (for [[x foo] [y bar]] baz) -> + (for* [x foo] + (for* [y bar] baz))" - ;; TODO: that signature sucks. - ;; (for [[x foo] [y bar]] baz) would be more consistent - (if (% (len args) 2) - (macro-error args "for needs an even number of elements in its first argument")) (if args - `(foreach [~(.pop args 0) ~(.pop args 0)] (for ~args ~@body)) + `(for* ~(.pop args 0) (for ~args ~@body)) + `(do ~@body))) + + +(defmacro with [args &rest body] + "shorthand for nested for* loops: + (with [[x foo] [y bar]] baz) -> + (with* [x foo] + (with* [y bar] + baz))" + (if args + `(with* ~(.pop args 0) (with ~args ~@body)) `(do ~@body))) @@ -71,7 +78,7 @@ (setv root (check-branch branch)) (setv latest-branch root) - (foreach [branch branches] + (for* [branch branches] (setv cur-branch (check-branch branch)) (.append latest-branch cur-branch) (setv latest-branch cur-branch)) @@ -81,7 +88,7 @@ (defmacro -> [head &rest rest] ;; TODO: fix the docstring by someone who understands this (setv ret head) - (foreach [node rest] + (for* [node rest] (if (not (isinstance node HyExpression)) (setv node `(~node))) (.insert node 1 ret) @@ -92,7 +99,7 @@ (defmacro ->> [head &rest rest] ;; TODO: fix the docstring by someone who understands this (setv ret head) - (foreach [node rest] + (for* [node rest] (if (not (isinstance node HyExpression)) (setv node `(~node))) (.append node ret) @@ -113,7 +120,7 @@ (defmacro yield-from [iterable] "Yield all the items from iterable" (let [[x (gensym)]] - `(foreach [~x ~iterable] + `(for* [~x ~iterable] (yield ~x)))) (defmacro with-gensyms [args &rest body] diff --git a/tests/__init__.py b/tests/__init__.py index bbd24e2..5b9f090 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,4 +12,5 @@ from .native_tests.when import * # noqa from .native_tests.with_decorator import * # noqa from .native_tests.core import * # noqa from .native_tests.reader_macros import * # noqa +from .native_tests.with_test import * # noqa from .native_tests.contrib.anaphoric import * # noqa diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index a362d66..ebcfabd 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -295,9 +295,9 @@ def test_ast_bad_assoc(): def test_ast_bad_with(): "Make sure AST can't compile invalid with" - cant_compile("(with)") - cant_compile("(with [])") - cant_compile("(with [] (pass))") + cant_compile("(with*)") + cant_compile("(with* [])") + cant_compile("(with* [] (pass))") def test_ast_valid_while(): @@ -305,14 +305,14 @@ def test_ast_valid_while(): can_compile("(while foo bar)") -def test_ast_valid_foreach(): - "Make sure AST can compile valid foreach" - can_compile("(foreach [a 2])") +def test_ast_valid_for(): + "Make sure AST can compile valid for" + can_compile("(for [[a 2]])") -def test_ast_invalid_foreach(): - "Make sure AST can't compile invalid foreach" - cant_compile("(foreach [a 1] (else 1 2))") +def test_ast_invalid_for(): + "Make sure AST can't compile invalid for" + cant_compile("(for* [a 1] (else 1 2))") def test_ast_expression_basics(): diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 4efe768..25ecc33 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -31,12 +31,12 @@ (defn test-for-loop [] "NATIVE: test for loops?" (setv count 0) - (for [x [1 2 3 4 5]] + (for [[x [1 2 3 4 5]]] (setv count (+ count x))) (assert (= count 15)) (setv count 0) - (for [x [1 2 3 4 5] - y [1 2 3 4 5]] + (for [[x [1 2 3 4 5]] + [y [1 2 3 4 5]]] (setv count (+ count x y))) (assert (= count 150))) @@ -394,9 +394,9 @@ (defn test-yield [] "NATIVE: test yielding" - (defn gen [] (for [x [1 2 3 4]] (yield x))) + (defn gen [] (for [[x [1 2 3 4]]] (yield x))) (setv ret 0) - (for [y (gen)] (setv ret (+ ret y))) + (for [[y (gen)]] (setv ret (+ ret y))) (assert (= ret 10))) @@ -439,37 +439,37 @@ (defn test-context [] "NATIVE: test with" - (with [fd (open "README.md" "r")] (assert fd)) - (with [(open "README.md" "r")] (do))) + (with [[fd (open "README.md" "r")]] (assert fd)) + (with [[(open "README.md" "r")]] (do))) (defn test-with-return [] "NATIVE: test that with returns stuff" (defn read-file [filename] - (with [fd (open filename "r")] (.read fd))) + (with [[fd (open filename "r")]] (.read fd))) (assert (!= 0 (len (read-file "README.md"))))) (defn test-for-doodle [] "NATIVE: test for-do" (do (do (do (do (do (do (do (do (do (setv (, x y) (, 0 0))))))))))) - (foreach [- [1 2]] + (for [[- [1 2]]] (do (setv x (+ x 1)) (setv y (+ y 1)))) (assert (= y x 2))) -(defn test-foreach-else [] - "NATIVE: test foreach else" +(defn test-for-else [] + "NATIVE: test for else" (let [[x 0]] - (foreach [a [1 2]] + (for* [a [1 2]] (setv x (+ x a)) (else (setv x (+ x 50)))) (assert (= x 53))) (let [[x 0]] - (foreach [a [1 2]] + (for* [a [1 2]] (setv x (+ x a)) (else)) (assert (= x 3)))) @@ -636,7 +636,7 @@ (defn test-nested-if [] "NATIVE: test nested if" - (for [x (range 10)] + (for [[x (range 10)]] (if (in "foo" "foobar") (do (if true true true)) @@ -800,16 +800,16 @@ (defn test-break-breaking [] "NATIVE: test checking if break actually breaks" - (defn holy-grail [] (for [x (range 10)] (if (= x 5) (break))) x) + (defn holy-grail [] (for [[x (range 10)]] (if (= x 5) (break))) x) (assert (= (holy-grail) 5))) (defn test-continue-continuation [] "NATIVE: test checking if continue actually continues" (setv y []) - (for [x (range 10)] - (if (!= x 5) - (continue)) + (for [[x (range 10)]] + (if (!= x 5) + (continue)) (.append y x)) (assert (= y [5]))) diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy index 911a9c9..0841b1e 100644 --- a/tests/native_tests/native_macros.hy +++ b/tests/native_tests/native_macros.hy @@ -56,7 +56,7 @@ (defn test-midtree-yield-in-for [] "NATIVE: test yielding in a for with a return" (defn kruft-in-for [] - (for [i (range 5)] + (for* [i (range 5)] (yield i)) (+ 1 2))) @@ -72,7 +72,7 @@ (defn test-multi-yield [] "NATIVE: testing multiple yields" (defn multi-yield [] - (for [i (range 3)] + (for* [i (range 3)] (yield i)) (yield "a") (yield "end")) @@ -97,7 +97,7 @@ (defn test-yield-from [] "NATIVE: testing yield from" (defn yield-from-test [] - (for [i (range 3)] + (for* [i (range 3)] (yield i)) (yield-from [1 2 3])) (assert (= (list (yield-from-test)) [0 1 2 1 2 3]))) diff --git a/tests/native_tests/with_test.hy b/tests/native_tests/with_test.hy new file mode 100644 index 0000000..b094f35 --- /dev/null +++ b/tests/native_tests/with_test.hy @@ -0,0 +1,44 @@ +(defclass WithTest [object] + [(--init-- + (fn [self val] + (setv self.val val) + None)) + + (--enter-- + (fn [self] + self.val)) + + (--exit-- + (fn [self type value traceback] + (setv self.val None)))]) + +(defn test-single-with [] + "NATIVE: test a single with" + (with [[t (WithTest 1)]] + (assert (= t 1)))) + +(defn test-two-with [] + "NATIVE: test two withs" + (with [[t1 (WithTest 1)] + [t2 (WithTest 2)]] + (assert (= t1 1)) + (assert (= t2 2)))) + +(defn test-thrice-with [] + "NATIVE: test three withs" + (with [[t1 (WithTest 1)] + [t2 (WithTest 2)] + [t3 (WithTest 3)]] + (assert (= t1 1)) + (assert (= t2 2)) + (assert (= t3 3)))) + +(defn test-quince-with [] + "NATIVE: test four withs, one with no args" + (with [[t1 (WithTest 1)] + [t2 (WithTest 2)] + [t3 (WithTest 3)] + [(WithTest 4)]] + (assert (= t1 1)) + (assert (= t2 2)) + (assert (= t3 3)))) From 4e3b6fd4cf6d36302bbc7f80c11ca3aa5ac5961b Mon Sep 17 00:00:00 2001 From: Bob Tolbert Date: Mon, 30 Dec 2013 14:42:55 -0700 Subject: [PATCH 3/9] Add some docs for gensym and siblings --- docs/language/api.rst | 71 +++++++++++++++++++++++++++ docs/language/internals.rst | 97 ++++++++++++++++++++++++++++++++++--- 2 files changed, 162 insertions(+), 6 deletions(-) diff --git a/docs/language/api.rst b/docs/language/api.rst index 9934b4d..12a7e06 100644 --- a/docs/language/api.rst +++ b/docs/language/api.rst @@ -357,6 +357,8 @@ Parameters may have following keywords in front of them: => (zig-zag-sum 1 2 3 4 5 6) -3 +.. _defmacro: + defmacro -------- @@ -378,6 +380,24 @@ between the operands. => (infix (1 + 1)) 2 +.. _defmacro/g!: + +defmacro/g! +------------ + +.. versionadded:: 0.9.12 + +`defmacro/g!` is a special version of `defmacro` that is used to +automatically generate :ref:`gensym` for any symbol that +starts with ``g!``. + +So ``g!a`` would become ``(gensym "a")``. + +.. seealso:: + + Section :ref:`using-gensym` + + del --- @@ -494,6 +514,28 @@ normally. If the execution is halted with `break`, the `else` does not execute. loop finished +.. _gensym: + +gensym +------ + +.. versionadded:: 0.9.12 + +`gensym` form is used to generate a unique symbol to allow writing macros +without accidental variable name clashes. + +.. code-block:: clj + + => (gensym) + u':G_1235' + + => (gensym "x") + u':x_1236' + +.. seealso:: + + Section :ref:`using-gensym` + get --- @@ -983,6 +1025,35 @@ values that are incremented by 1. When decorated `addition` is called with value 4 +.. _with-gensyms: + +with-gensyms +------------- + +.. versionadded:: 0.9.12 + +`with-gensym` form is used to generate a set of :ref:`gensym` for use +in a macro. + +.. code-block:: clojure + + (with-gensyms [a b c] + ...) + +expands to: + +.. code-block:: clojure + + (let [[a (gensym) + [b (gensym) + [c (gensym)]] + ...) + +.. seealso:: + + Section :ref:`using-gensym` + + yield ----- diff --git a/docs/language/internals.rst b/docs/language/internals.rst index ccf8263..cf4f99c 100644 --- a/docs/language/internals.rst +++ b/docs/language/internals.rst @@ -2,26 +2,111 @@ Internal Hy Documentation ========================= -.. info:: - These bits are for folks who hack on Hy it's self, mostly! +.. note:: + These bits are for folks who hack on Hy itself, mostly! Hy Models ========= -.. TODO:: +.. todo:: Write this. Hy Macros ========= -.. TODO:: - Write this. +.. _using-gensym: + +Using gensym for safer macros +------------------------------ + +When writing macros, one must be careful to avoid capturing external variables +or using variable names that might conflict with user code. + +We will use an example macro ``nif`` (see http://letoverlambda.com/index.cl/guest/chap3.html#sec_5 +for a more complete description.) ``nif`` is an example, something like a numeric ``if``, +where based on the expression, one of the 3 forms is called depending on if the +expression is positive, zero or negative. + +A first pass might be someting like: + +.. code-block:: clojure + + (defmacro nif [expr pos-form zero-form neg-form] + `(let [[obscure-name ~expr]] + (cond [(pos? obscure-name) ~pos-form] + [(zero? obscure-name) ~zero-form] + [(neg? obscure-name) ~neg-form]))) + +where ``obsure-name`` is an attempt to pick some variable name as not to +conflict with other code. But of course, while well-intentioned, +this is no guarantee. + +The method :ref:`gensym` is designed to generate a new, unique symbol for just +such an occasion. A much better version of ``nif`` would be: + +.. code-block:: clojure + + (defmacro nif [expr pos-form zero-form neg-form] + (let [[g (gensym)]] + `(let [[~g ~expr]] + (cond [(pos? ~g) ~pos-form] + [(zero? ~g) ~zero-form] + [(neg? ~g) ~neg-form])))) + +This is an easy case, since there is only one symbol. But if there is +a need for several gensym's there is a second macro :ref:`with-gensyms` that +basically expands to a series of ``let`` statements: + +.. code-block:: clojure + + (with-gensyms [a b c] + ...) + +expands to: + +.. code-block:: clojure + + (let [[a (gensym) + [b (gensym) + [c (gensym)]] + ...) + +so our re-written ``nif`` would look like: + +.. code-block:: clojure + + (defmacro nif [expr pos-form zero-form neg-form] + (with-gensyms [g] + `(let [[~g ~expr]] + (cond [(pos? ~g) ~pos-form] + [(zero? ~g) ~zero-form] + [(neg? ~g) ~neg-form])))) + +Finally, though we can make a new macro that does all this for us. :ref:`defmacro/g!` +will take all symbols that begin with ``g!`` and automatically call ``gensym`` with the +remainder of the symbol. So ``g!a`` would become ``(gensym "a")``. + +Our final version of ``nif``, built with ``defmacro/g!`` becomes: + +.. code-block:: clojure + + (defmacro/g! nif [expr pos-form zero-form neg-form] + `(let [[~g!res ~expr]] + (cond [(pos? ~g!res) ~pos-form] + [(zero? ~g!res) ~zero-form] + [(neg? ~g!res) ~neg-form])))) + + + +Checking macro arguments and raising exceptions +----------------------------------------------- + Hy Compiler Builtins ==================== -.. TODO:: +.. todo:: Write this. From d7956d03c35d23407172a69663576b5d95a4f892 Mon Sep 17 00:00:00 2001 From: Bob Tolbert Date: Mon, 30 Dec 2013 15:09:17 -0700 Subject: [PATCH 4/9] Adding documentation for flatten --- docs/language/core.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/language/core.rst b/docs/language/core.rst index acdc025..9b24747 100644 --- a/docs/language/core.rst +++ b/docs/language/core.rst @@ -575,6 +575,26 @@ See also :ref:`remove-fn`. => (list (filter even? [1 2 3 -4 5 -7])) [2, -4] +.. _flatten-fn: + +flatten +------- + +.. versionadded:: 0.9.12 + +Usage: ``(flatten coll)`` + +Return a single list of all the items in ``coll``, by flattening all +contained lists and/or tuples. + +.. code-block:: clojure + + => (flatten [1 2 [3 4] 5]) + [1, 2, 3, 4, 5] + + => (flatten ["foo" (, 1 2) [1 [2 3] 4] "bar"]) + ['foo', 1, 2, 1, 2, 3, 4, 'bar'] + .. _iterate-fn: From 67fd0ddbbeb7f6418cb151c1b4217dd29eafc62e Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Mon, 30 Dec 2013 17:32:57 -0500 Subject: [PATCH 5/9] Document the compiler a little. --- docs/language/internals.rst | 167 ++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/docs/language/internals.rst b/docs/language/internals.rst index cf4f99c..df43061 100644 --- a/docs/language/internals.rst +++ b/docs/language/internals.rst @@ -13,6 +13,173 @@ Hy Models Write this. +Hy Internal Theory +================== + +.. _overview: + +Overview +-------- + +The Hy internals work by acting as a front-end to Python bytecode, so that +Hy it's self compiles down to Python Bytecode, allowing an unmodified Python +runtime to run Hy. + +The way we do this is by translating Hy into Python AST, and building that AST +down into Python bytecode using standard internals, so that we don't have +to duplicate all the work of the Python internals for every single Python +release. + +Hy works in four stages. The following sections will cover each step of Hy +from source to runtime. + +.. _lexing: + +Lexing / tokenizing +------------------- + +The first stage of compiling hy is to lex the source into tokens that we can +deal with. We use a project called rply, which is a really nice (and fast) +parser, written in a subset of Python called rply. + +The lexing code is all defined in ``hy.lex.lexer``. This code is mostly just +defining the Hy grammer, and all the actual hard parts are taken care of by +rply -- we just define "callbacks" for rply in ``hy.lex.parser``, which take +the tokens generated, and return the Hy models. + +You can think of the Hy models as the "AST" for Hy, it's what Macros operate +on (directly), and it's what the compiler uses when it compiles Hy down. + +Check the documentation for more information on the Hy models for more +information regarding the Hy models, and what they mean. + +.. TODO: Uh, we should, like, document models. + + +.. _compiling: + +Compiling +--------- + +This is where most of the magic in Hy happens. This is where we take Hy AST +(the models), and compile them into Python AST. A couple of funky things happen +here to work past a few problems in AST, and working in the compiler is some +of the most important work we do have. + +The compiler is a bit complex, so don't feel bad if you don't grok it on the +first shot, it may take a bit of time to get right. + +The main entry-point to the Compiler is ``HyASTCompiler.compile``. This method +is invoked, and the only real "public" method on the class (that is to say, +we don't really promise the API beyond that method). + +In fact, even internally, we don't recurse directly hardly ever, we almost +always force the Hy tree through ``compile``, and will often do this with +sub-elements of an expression that we have. It's up to the Type-based dispatcher +to properly dispatch sub-elements. + +All methods that preform a compilation are marked with the ``@builds()`` +decorator. You can either pass the class of the Hy model that it compiles, +or you can use a string for expressions. I'll clear this up in a second. + +First stage type-dispatch +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's start in the ``compile`` method. The first thing we do is check the +Type of the thing we're building. We look up to see if we have a method that +can build the ``type()`` that we have, and dispatch to the method that can +handle it. If we don't have any methods that can build that type, we raise +an internal ``Exception``. + +For instance, if we have a ``HyString``, we have an almost 1-to-1 mapping of +Hy AST to Python AST. The ``compile_string`` method takes the ``HyString``, and +returns an ``ast.Str()`` that's populated with the correct line-numbers and +content. + +Macro-expand +~~~~~~~~~~~~ + +If we get a ``HyExpression``, we'll attempt to see if this is a known +Macro, and push to have it expanded by invoking ``hy.macros.macroexpand``, then +push the result back into ``HyASTCompiler.compile``. + +Second stage expression-dispatch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The only special case is the ``HyExpression``, since we need to create different +AST depending on the special form in question. For instance, when we hit an +``(if true true false)``, we need to generate a ``ast.If``, and properly +compile the sub-nodes. This is where the ``@builds()`` with a String as an +argument comes in. + +For the ``compile_expression`` (which is defined with an +``@builds(HyExpression)``) will dispatch based on the string of the first +argument. If, for some reason, the first argument is not a string, it will +properly handle that case as well (most likely by raising an ``Exception``). + +If the String isn't known to Hy, it will default to create an ``ast.Call``, +which will try to do a runtime call (in Python, something like ``foo()``). + +Issues hit with Python AST +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Python AST is great; it's what's enabled us to write such a powerful project +on top of Python without having to fight Python too hard. Like anything, we've +had our fair share of issues, and here's a short list of the common ones you +might run into. + +*Python differentiates between Statements and Expressions*. + +This might not sound like a big deal -- in fact, to most Python programmers, +this will shortly become a "Well, yeah" moment. + +In Python, doing something like: + +``print for x in range(10): pass``, because ``print`` prints expressions, and +``for`` isn't an expression, it's a control flow statement. Things like +``1 + 1`` are Expressions, as is ``lambda x: 1 + x``, but other language +features, such as ``if``, ``for``, or ``while`` are statements. + +Since they have no "value" to Python, this makes working in Hy hard, since +doing something like ``(print (if true true false))`` is not just common, it's +expected. + +As a result, we auto-mangle things using a ``Result`` object, where we offer +up any ``ast.stmt`` that need to get run, and a single ``ast.expr`` that can +be used to get the value of whatever was just run. Hy does this by forcing +assignment to things while running. + +As example, the Hy:: + + (print (if true true false)) + +Will turn into:: + + if True: + _mangled_name_here = True + else: + _mangled_name_here = False + + print _mangled_name_here + + +OK, that was a bit of a lie, since we actually turn that statement +into:: + + print True if True else False + +By forcing things into an ``ast.expr`` if we can, but the general idea holds. + + +Runtime +------- + +After we have a Python AST tree that's complete, we can try and compile it to +Python bytecode by pushing it through ``eval``. From here on out, we're no +longer in control, and Python is taking care of everything. This is why things +like Python tracebacks, pdb and django apps work. + + Hy Macros ========= From b011048b419d32135bfe20cf67aa9347cca1e022 Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Mon, 30 Dec 2013 18:02:03 -0500 Subject: [PATCH 6/9] allow one-shot merge --- docs/hacking.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/hacking.rst b/docs/hacking.rst index 6ee479c..0a35898 100644 --- a/docs/hacking.rst +++ b/docs/hacking.rst @@ -100,6 +100,9 @@ If a core member is sending in a PR, please find 2 core members that don't include them PR submitter. The idea here is that one can work with the PR author, and a second acks the entire change set. +If the change is adding documentation, feel free to just merge after one +ACK. We've got low coverage, so it'd be great to keep that barrier low. + Core Team ========= From 426d34288f37c61182c7c0ef054a127dfbd83c6d Mon Sep 17 00:00:00 2001 From: Foxboron Date: Tue, 31 Dec 2013 02:06:51 +0100 Subject: [PATCH 7/9] Added docs and one small bug fix in defreader --- docs/language/api.rst | 19 +++++++++++ docs/language/index.rst | 1 + docs/language/readermacros.rst | 61 ++++++++++++++++++++++++++++++++++ hy/compiler.py | 2 +- 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 docs/language/readermacros.rst diff --git a/docs/language/api.rst b/docs/language/api.rst index aff6745..4e70b7d 100644 --- a/docs/language/api.rst +++ b/docs/language/api.rst @@ -397,6 +397,25 @@ So ``g!a`` would become ``(gensym "a")``. Section :ref:`using-gensym` +defreader +--------- + +.. versionadded:: 0.9.12 + +`defreader` defines a reader macro, enabling you to restructure or +modify syntax. + +.. code-block:: clj + + => (defreader ^ [expr] (print expr)) + => #^(1 2 3 4) + (1 2 3 4) + => #^"Hello" + "Hello" + +.. seealso:: + + Section :ref:`Reader Macros ` del --- diff --git a/docs/language/index.rst b/docs/language/index.rst index 5132647..4721eb3 100644 --- a/docs/language/index.rst +++ b/docs/language/index.rst @@ -11,3 +11,4 @@ Contents: api core internals + readermacros diff --git a/docs/language/readermacros.rst b/docs/language/readermacros.rst new file mode 100644 index 0000000..0c05d9c --- /dev/null +++ b/docs/language/readermacros.rst @@ -0,0 +1,61 @@ +.. _reader-macros: + +.. highlight:: clj + +============= +Reader Macros +============= + +Reader macros gives LISP the power to modify and alter syntax on the fly. +You don't want polish notation? A reader macro can easily do just that. Want +Clojure's way of having a regex? Reader macros can also do this easily. + + +Syntax +====== + +:: + + => (defreader ^ [expr] (print expr)) + => #^(1 2 3 4) + (1 2 3 4) + => #^"Hello" + "Hello" + => #^1+2+3+4+3+2 + 1+2+3+4+3+2 + + +Implementation +============== + +Hy uses ``defreader`` to define the reader symbol, and ``#`` as the dispatch +character. ``#`` expands into ``(dispatch_reader_macro ...)`` where the symbol +and expression is quoted, and then passed along to the correct function:: + + => (defreader ^ ...) + => #^() + ;=> (dispatch_reader_macro '^ '()) + + +``defreader`` takes a single character as symbol name for the reader macro, +anything longer will return an error. Implementation wise, ``defreader`` +expands into a lambda covered with a decorator, this decorater saves the +lambda in a dict with its module name and symbol. + +:: + + => (defreader ^ [expr] (print expr)) + ;=> (with_decorator (hy.macros.reader ^) (fn [expr] (print expr))) + + +Anything passed along is quoted, thus given to the function defined. + +:: + + => #^"Hello" + "Hello" + +.. warning:: + Because of a limitation in Hy's lexer and parser, reader macros can't + redefine defined syntax such as ``()[]{}``. This will most likely be + adressed in the future. diff --git a/hy/compiler.py b/hy/compiler.py index 9a3b710..8b34a9d 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1866,7 +1866,7 @@ class HyASTCompiler(object): expression.pop(0) name = expression.pop(0) NOT_READERS = [":", "&"] - if name in NOT_READERS: + if name in NOT_READERS or len(name) > 1: raise NameError("%s can't be used as a macro reader symbol" % name) if not isinstance(name, HySymbol): raise HyTypeError(name, From 9b1990901c619d6f8e728d3bed61607dd9d06e83 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Tue, 31 Dec 2013 15:51:58 +0200 Subject: [PATCH 8/9] Fix |Unknown directive type "todo"| error. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 40eb062..6b45309 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ import hy # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] +extensions = ['sphinx.ext.todo'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From 931ce889f9912f75b483d5c7b877976dbab05504 Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Tue, 31 Dec 2013 11:26:31 -0500 Subject: [PATCH 9/9] s/rply/rpython/ --- docs/language/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/language/internals.rst b/docs/language/internals.rst index df43061..b9513f6 100644 --- a/docs/language/internals.rst +++ b/docs/language/internals.rst @@ -40,7 +40,7 @@ Lexing / tokenizing The first stage of compiling hy is to lex the source into tokens that we can deal with. We use a project called rply, which is a really nice (and fast) -parser, written in a subset of Python called rply. +parser, written in a subset of Python called rpython. The lexing code is all defined in ``hy.lex.lexer``. This code is mostly just defining the Hy grammer, and all the actual hard parts are taken care of by