Merge pull request #1310 from gilch/tagged-literals

Tag macros

NEWS

@@ -1,3 +1,9 @@
+Changes from 0.13.0
+
+   [ Language Changes ]
+   * Single-character "sharp macros" changed to "tag macros", which can have
+     longer names
+
 Changes from 0.12.1
 
    [ Language Changes ]

docs/language/api.rst

@@ -779,25 +779,21 @@ For example,
     42
 
 
-defsharp
+deftag
 --------
 
 .. versionadded:: 0.13.0
 
-``defsharp`` defines a sharp macro. A sharp macro is a unary macro that has the
+``deftag`` defines a tag macro. A tag macro is a unary macro that has the
-same semantics as an ordinary macro defined with ``defmacro``, but can be
+same semantics as an ordinary macro defined with ``defmacro``. It is called with
-called without parentheses and with less whitespace. The name of a sharp macro
+the syntax ``#tag FORM``, where ``tag`` is the name of the macro, and ``FORM``
-must be exactly one character long. It is called with the syntax ``#cFORM``,
+is any form. The ``tag`` is often only one character, but it can be any symbol.
-where ``#`` is a literal sharp sign (hence the term "sharp macro"), ``c`` is
-the name of the macro, and ``FORM`` is any form. Whitspace is forbidden between
-``#`` and ``c``. Whitespace is allowed between ``c`` and ``FORM``, but not
-required.
 
 .. code-block:: clj
 
-    => (defsharp ♣ [expr] `[~expr ~expr])
+    => (deftag ♣ [expr] `[~expr ~expr])
     <function <lambda> at 0x7f76d0271158>
-    => #♣5
+    => #♣ 5
     [5, 5]
     => (setv x 0)
     => #♣(+= x 1)
@@ -805,13 +801,13 @@ required.
     => x
     2
 
-In this example, if you used ``(defmacro ♣ ...)`` instead of ``(defsharp
+In this example, if you used ``(defmacro ♣ ...)`` instead of ``(deftag
 ♣ ...)``, you would call the macro as ``(♣ 5)`` or ``(♣ (+= x 1))``.
 
-The syntax for calling sharp macros is similar to that of reader macros a la
+The syntax for calling tag macros is similar to that of reader macros a la
-Common Lisp's ``SET-MACRO-CHARACTER``. In fact, before Hy 0.13.0, sharp macros
+Common Lisp's ``SET-MACRO-CHARACTER``. In fact, before Hy 0.13.0, tag macros
 were called "reader macros", and defined with ``defreader`` rather than
-``defsharp``. True reader macros are not (yet) implemented in Hy.
+``deftag``. True reader macros are not (yet) implemented in Hy.
 
 del
 ---
@@ -1731,7 +1727,7 @@ will be 4 (``1+1 + 1+1``).
 
 .. versionadded:: 0.12.0
 
-The sharp macro ``#@`` can be used as a shorthand for ``with-decorator``. With
+The tag macro ``#@`` can be used as a shorthand for ``with-decorator``. With
 ``#@``, the previous example becomes:
 
 .. code-block:: clj

docs/tutorial.rst

@@ -582,13 +582,13 @@ elements, so by the time program started executing, it actually reads:
   (+ 1 2 3)
 
 Sometimes it's nice to be able to call a one-parameter macro without
-parentheses. Sharp macros allow this. The name of a sharp macro must be only
+parentheses. Tag macros allow this. The name of a tag macro is typically
 one character long, but since Hy operates well with Unicode, we aren't running
 out of characters that soon:
 
 .. code-block:: clj
 
-  => (defsharp ↻ [code]
+  => (deftag ↻ [code]
   ...  (setv op (last code) params (list (butlast code)))
   ...  `(~op ~@params))
   => #↻(1 2 3 +)

hy/compiler.py

@@ -13,7 +13,7 @@ from hy.lex.parser import hy_symbol_mangle
 import hy.macros
 from hy._compat import (
     str_type, bytes_type, long_type, PY3, PY34, PY35, raise_empty)
-from hy.macros import require, macroexpand, sharp_macroexpand
+from hy.macros import require, macroexpand, tag_macroexpand
 import hy.importer
 
 import traceback
@@ -2465,20 +2465,20 @@ class HyASTCompiler(object):
 
         return ret
 
-    @builds("defsharp")
+    @builds("deftag")
     @checkargs(min=2)
-    def compile_sharp_macro(self, expression):
+    def compile_tag_macro(self, expression):
         expression.pop(0)
         name = expression.pop(0)
-        if name == ":" or name == "&" or len(name) > 1:
+        if name == ":" or name == "&":
-            raise NameError("%s can't be used as a sharp macro name" % name)
+            raise NameError("%s can't be used as a tag macro name" % name)
         if not isinstance(name, HySymbol) and not isinstance(name, HyString):
             raise HyTypeError(name,
                               ("received a `%s' instead of a symbol "
-                               "for sharp macro name" % type(name).__name__))
+                               "for tag macro name" % type(name).__name__))
         name = HyString(name).replace(name)
         new_expression = HyExpression([
-            HyExpression([HySymbol("hy.macros.sharp"), name]),
+            HyExpression([HySymbol("hy.macros.tag"), name]),
             HyExpression([HySymbol("fn")] + expression),
         ]).replace(expression)
 
@@ -2486,18 +2486,19 @@ class HyASTCompiler(object):
 
         return ret
 
-    @builds("dispatch_sharp_macro")
+    @builds("dispatch_tag_macro")
     @checkargs(exact=2)
-    def compile_dispatch_sharp_macro(self, expression):
+    def compile_dispatch_tag_macro(self, expression):
-        expression.pop(0)  # dispatch-sharp-macro
+        expression.pop(0)  # dispatch-tag-macro
-        str_char = expression.pop(0)
+        tag = expression.pop(0)
-        if not type(str_char) == HyString:
+        if not type(tag) == HyString:
             raise HyTypeError(
-                str_char,
+                tag,
-                "Trying to expand a sharp macro using `{0}' instead "
+                "Trying to expand a tag macro using `{0}' instead "
-                "of string".format(type(str_char).__name__),
+                "of string".format(type(tag).__name__),
             )
-        expr = sharp_macroexpand(str_char, expression.pop(0), self)
+        tag = HyString(hy_symbol_mangle(str(tag))).replace(tag)
+        expr = tag_macroexpand(tag, expression.pop(0), self)
         return self.compile(expr)
 
     @builds("eval_and_compile")

hy/completer.py

@@ -40,11 +40,11 @@ class Completer(object):
                      builtins.__dict__,
                      hy.macros._hy_macros[None],
                      namespace]
-        self.sharp_path = [hy.macros._hy_sharp[None]]
+        self.tag_path = [hy.macros._hy_tag[None]]
         if '__name__' in namespace:
             module_name = namespace['__name__']
             self.path.append(hy.macros._hy_macros[module_name])
-            self.sharp_path.append(hy.macros._hy_sharp[module_name])
+            self.tag_path.append(hy.macros._hy_tag[module_name])
 
     def attr_matches(self, text):
         # Borrowed from IPython's completer
@@ -81,10 +81,10 @@ class Completer(object):
                         matches.append(k)
         return matches
 
-    def sharp_matches(self, text):
+    def tag_matches(self, text):
         text = text[1:]
         matches = []
-        for p in self.sharp_path:
+        for p in self.tag_path:
             for k in p.keys():
                 if isinstance(k, string_types):
                     if k.startswith(text):
@@ -93,7 +93,7 @@ class Completer(object):
 
     def complete(self, text, state):
         if text.startswith("#"):
-            matches = self.sharp_matches(text)
+            matches = self.tag_matches(text)
         elif "." in text:
             matches = self.attr_matches(text)
         else:

hy/core/macros.hy

@@ -230,7 +230,7 @@
        (sys.exit ~retval))))
 
 
-(defsharp @ [expr]
+(deftag @ [expr]
   (setv decorators (cut expr None -1)
         fndef (get expr -1))
   `(with-decorator ~@decorators ~fndef))

hy/lex/lexer.py

@@ -12,6 +12,7 @@ lg = LexerGenerator()
 # i.e. a space or a closing brace/paren/curly
 end_quote = r'(?![\s\)\]\}])'
 
+identifier = r'[^()\[\]{}\'"\s;]+'
 
 lg.add('LPAREN', r'\(')
 lg.add('RPAREN', r'\)')
@@ -25,7 +26,7 @@ lg.add('QUASIQUOTE', r'`%s' % end_quote)
 lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
 lg.add('UNQUOTE', r'~%s' % end_quote)
 lg.add('HASHBANG', r'#!.*[^\r\n]')
-lg.add('HASHOTHER', r'#[^{]')
+lg.add('HASHOTHER', r'#%s' % identifier)
 
 # A regexp which matches incomplete strings, used to support
 # multi-line strings in the interpreter
@@ -44,7 +45,7 @@ partial_string = r'''(?x)
 lg.add('STRING', r'%s"' % partial_string)
 lg.add('PARTIAL_STRING', partial_string)
 
-lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+')
+lg.add('IDENTIFIER', identifier)
 
 
 lg.ignore(r';.*(?=\r|\n|$)')

hy/lex/parser.py

@@ -200,10 +200,11 @@ def term_unquote_splice(p):
 @pg.production("term : HASHOTHER term")
 @set_quote_boundaries
 def hash_other(p):
-    st = p[0].getstr()[1]
+    # p == [(Token('HASHOTHER', '#foo'), bar)]
+    st = p[0].getstr()[1:]
     str_object = HyString(st)
     expr = p[1]
-    return HyExpression([HySymbol("dispatch_sharp_macro"), str_object, expr])
+    return HyExpression([HySymbol("dispatch_tag_macro"), str_object, expr])
 
 
 @pg.production("set : HLCURLY list_contents RCURLY")

hy/macros.py

@@ -18,7 +18,7 @@ EXTRA_MACROS = [
 ]
 
 _hy_macros = defaultdict(dict)
-_hy_sharp = defaultdict(dict)
+_hy_tag = defaultdict(dict)
 
 
 def macro(name):
@@ -50,8 +50,8 @@ def macro(name):
     return _
 
 
-def sharp(name):
+def tag(name):
-    """Decorator to define a sharp macro called `name`.
+    """Decorator to define a tag macro called `name`.
 
     This stores the macro `name` in the namespace for the module where it is
     defined.
@@ -59,14 +59,14 @@ def sharp(name):
     If the module where it is defined is in `hy.core`, then the macro is stored
     in the default `None` namespace.
 
-    This function is called from the `defsharp` special form in the compiler.
+    This function is called from the `deftag` special form in the compiler.
 
     """
     def _(fn):
         module_name = fn.__module__
         if module_name.startswith("hy.core"):
             module_name = None
-        _hy_sharp[module_name][name] = fn
+        _hy_tag[module_name][name] = fn
 
         return fn
     return _
@@ -90,7 +90,7 @@ def require(source_module, target_module,
     if prefix:
         prefix += "."
 
-    for d in _hy_macros, _hy_sharp:
+    for d in _hy_macros, _hy_tag:
         for name, macro in d[source_module].items():
             seen_names.add(name)
             if all_macros:
@@ -210,19 +210,19 @@ def macroexpand_1(tree, compiler):
     return tree
 
 
-def sharp_macroexpand(char, tree, compiler):
+def tag_macroexpand(tag, tree, compiler):
-    """Expand the sharp macro "char" with argument `tree`."""
+    """Expand the tag macro "tag" with argument `tree`."""
     load_macros(compiler.module_name)
 
-    sharp_macro = _hy_sharp[compiler.module_name].get(char)
+    tag_macro = _hy_tag[compiler.module_name].get(tag)
-    if sharp_macro is None:
+    if tag_macro is None:
         try:
-            sharp_macro = _hy_sharp[None][char]
+            tag_macro = _hy_tag[None][tag]
         except KeyError:
             raise HyTypeError(
-                char,
+                tag,
-                "`{0}' is not a defined sharp macro.".format(char)
+                "`{0}' is not a defined tag macro.".format(tag)
             )
 
-    expr = sharp_macro(tree)
+    expr = tag_macro(tree)
     return replace_hy_obj(wrap_value(expr), tree)

tests/macros/test_tag_macros.py

@@ -7,10 +7,10 @@ from hy.compiler import HyTypeError, HyASTCompiler
 from hy.lex import tokenize
 
 
-def test_sharp_macro_error():
+def test_tag_macro_error():
     """Check if we get correct error with wrong dispatch character"""
     try:
-        macroexpand(tokenize("(dispatch_sharp_macro '- '())")[0],
+        macroexpand(tokenize("(dispatch_tag_macro '- '())")[0],
                     HyASTCompiler(__name__))
     except HyTypeError as e:
         assert "with the character `-`" in str(e)

tests/native_tests/sharp_macros.hy

@@ -1,83 +0,0 @@
-;; Copyright 2017 the authors.
-;; This file is part of Hy, which is free software licensed under the Expat
-;; license. See the LICENSE.
-
-(import [functools [wraps]])
-
-
-(defn test-sharp-macro []
-  "Test a basic sharp macro"
-  (defsharp ^ [expr]
-    expr)
-
-  (assert (= #^"works" "works")))
-
-
-(defn test-sharp-macro-expr []
-  "Test basic exprs like lists and arrays"
-  (defsharp n [expr]
-    (get expr 1))
-
-  (assert (= #n[1 2] 2))
-  (assert (= #n(1 2) 2)))
-
-
-(defn test-sharp-macro-override []
-  "Test if we can override function symbols"
-  (defsharp + [n]
-    (+ n 1))
-
-  (assert (= #+2 3)))
-
-
-(defn test-sharp-macros-macros []
-  "Test if defsharp is actually a macro"
-  (defsharp t [expr]
-    `(, ~@expr))
-
-  (def a #t[1 2 3])
-
-  (assert (= (type a) tuple))
-  (assert (= (, 1 2 3) a)))
-
-
-(defn test-sharp-macro-string-name []
-  "Test if defsharp accepts a string as a macro name."
-
-  (defsharp "." [expr]
-    expr)
-
-  (assert (= #."works" "works")))
-
-
-(defn test-builtin-decorator-sharp []
-  (defn increment-arguments [func]
-    "Increments each argument passed to the decorated function."
-    ((wraps func)
-       (fn [&rest args &kwargs kwargs]
-         (apply func
-                (map inc args)
-                (dict-comp k (inc v) [[k v] (.items kwargs)])))))
-
-  #@(increment-arguments
-     (defn foo [&rest args &kwargs kwargs]
-       "Bar."
-       (, args kwargs)))
-
-  ;; The decorator did what it was supposed to
-  (assert (= (, (, 2 3 4) {"quux" 5 "baz" 6})
-             (foo 1 2 3 :quux 4 :baz 5)))
-
-  ;; @wraps preserved the docstring and __name__
-  (assert (= "foo" (. foo --name--)))
-  (assert (= "Bar." (. foo --doc--)))
-
-  ;; We can use the #@ sharp macro to apply more than one decorator
-  #@(increment-arguments
-     increment-arguments
-     (defn double-foo [&rest args &kwargs kwargs]
-       "Bar."
-       (, args kwargs)))
-
-  (assert (= (, (, 3 4 5) {"quux" 6 "baz" 7})
-             (double-foo 1 2 3 :quux 4 :baz 5))))

tests/native_tests/tag_macros.hy

@@ -0,0 +1,128 @@
+;; Copyright 2017 the authors.
+;; This file is part of Hy, which is free software licensed under the Expat
+;; license. See the LICENSE.
+
+(import [functools [wraps]])
+
+
+(defn test-tag-macro []
+  "Test a basic tag macro"
+  (deftag ^ [expr]
+    expr)
+
+  (assert (= #^"works" "works")))
+
+
+(defn test-long-tag-macro []
+  "Test a tag macro with a name longer than one character"
+  (deftag foo [expr]
+    `['foo ~expr])
+  (assert (= #foo'bar ['foo 'bar]))
+  (assert (= #foo"baz" ['foo "baz"]))
+  (assert (= #foo(- 44 2) ['foo 42]))
+  (assert (= #foo(, 42) ['foo (, 42)]))
+  (assert (= #foo[42] ['foo [42]]))
+  (assert (= #foo{4 2} ['foo {4 2}])))
+
+(defn test-hyphenated-tag-macro []
+  "Test if hyphens translate properly"
+  (deftag foo-bar [x]
+    `['foo ~x 'bar])
+  (assert (= #foo-bar 42) ['foo 42 'bar])
+  (assert (= #foo_bar 42) ['foo 42 'bar])
+  (deftag spam_eggs [x]
+    `['spam ~x 'eggs])
+  (assert (= #spam-eggs 42 ['spam 42 'eggs]))
+  (assert (= #spam_eggs 42 ['spam 42 'eggs])))
+
+
+(defn test-tag-macro-whitespace []
+  "Test whitespace after a tag macro"
+  (deftag foo [expr]
+    `['foo ~expr])
+  (assert (= #foo 42) ['foo 42])
+  (assert (= #foo (- 44 2) ['foo 42]))
+  (deftag b [x]
+    `['bar ~x])
+  (assert (= #b 42) ['bar 42])
+  ; # is allowed in tags, so this must be separated
+  (assert (= #b #{42} ['bar #{42}]))
+  ; multiple tags must likewise be separated
+  (assert (= #b #foo 42 ['bar ['foo 42]]))
+  ; newlines are also whitespace
+  (assert (= #foo
+
+             42 ['foo 42]))
+  (assert (= #foo; a semicolon/comment should count as whitespace
+             42
+             ['foo 42])))
+
+
+(defn test-tag-macro-expr []
+  "Test basic exprs like lists and arrays"
+  (deftag n [expr]
+    (get expr 1))
+
+  (assert (= #n[1 2] 2))
+  (assert (= #n(1 2) 2)))
+
+
+(defn test-tag-macro-override []
+  "Test if we can override function symbols"
+  (deftag + [n]
+    (+ n 1))
+
+  (assert (= #+2 3)))
+
+
+(defn test-tag-macros-macros []
+  "Test if deftag is actually a macro"
+  (deftag t [expr]
+    `(, ~@expr))
+
+  (def a #t[1 2 3])
+
+  (assert (= (type a) tuple))
+  (assert (= (, 1 2 3) a)))
+
+
+(defn test-tag-macro-string-name []
+  "Test if deftag accepts a string as a macro name."
+
+  (deftag "." [expr]
+    expr)
+
+  (assert (= #."works" "works")))
+
+
+(defn test-builtin-decorator-tag []
+  (defn increment-arguments [func]
+    "Increments each argument passed to the decorated function."
+    ((wraps func)
+       (fn [&rest args &kwargs kwargs]
+         (apply func
+                (map inc args)
+                (dict-comp k (inc v) [[k v] (.items kwargs)])))))
+
+  #@(increment-arguments
+     (defn foo [&rest args &kwargs kwargs]
+       "Bar."
+       (, args kwargs)))
+
+  ;; The decorator did what it was supposed to
+  (assert (= (, (, 2 3 4) {"quux" 5 "baz" 6})
+             (foo 1 2 3 :quux 4 :baz 5)))
+
+  ;; @wraps preserved the docstring and __name__
+  (assert (= "foo" (. foo --name--)))
+  (assert (= "Bar." (. foo --doc--)))
+
+  ;; We can use the #@ tag macro to apply more than one decorator
+  #@(increment-arguments
+     increment-arguments
+     (defn double-foo [&rest args &kwargs kwargs]
+       "Bar."
+       (, args kwargs)))
+
+  (assert (= (, (, 3 4 5) {"quux" 6 "baz" 7})
+             (double-foo 1 2 3 :quux 4 :baz 5))))

tests/test_lex.py

@@ -303,10 +303,10 @@ def test_complex():
     assert entry == HySymbol("j")
 
 
-def test_sharp_macro():
+def test_tag_macro():
-    """Ensure sharp macros are handled properly"""
+    """Ensure tag macros are handled properly"""
     entry = tokenize("#^()")
-    assert entry[0][0] == HySymbol("dispatch_sharp_macro")
+    assert entry[0][0] == HySymbol("dispatch_tag_macro")
     assert entry[0][1] == HyString("^")
     assert len(entry[0]) == 3