Add #[DELIM[ … ]DELIM] syntax for string literals

hy/lex/lexer.py

@@ -27,6 +27,12 @@ lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
 lg.add('UNQUOTE', r'~%s' % end_quote)
 lg.add('DISCARD', r'#_')
 lg.add('HASHSTARS', r'#\*+')
+lg.add('BRACKETSTRING', r'''(?x)
+    \# \[ ( [^\[\]]* ) \[    # Opening delimiter
+    \n?                      # A single leading newline will be ignored
+    ((?:\n|.)*?)             # Content of the string
+    \] \1 \]                 # Closing delimiter
+    ''')
 lg.add('HASHOTHER', r'#%s' % identifier)
 
 # A regexp which matches incomplete strings, used to support

hy/lex/parser.py

@@ -281,6 +281,17 @@ def t_partial_string(p):
     raise PrematureEndOfInput("Premature end of input")
 
 
+bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING')
+@pg.production("string : BRACKETSTRING")
+@set_boundaries
+def t_bracket_string(p):
+    m = bracket_string_re.match(p[0].value)
+    delim, content = m.groups()
+    s = HyString(content)
+    s.brackets = delim
+    return s
+
+
 @pg.production("identifier : IDENTIFIER")
 @set_boundaries
 def t_identifier(p):

tests/compilers/test_ast.py

@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 # Copyright 2017 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
@@ -46,6 +47,10 @@ def cant_compile(expr):
         return e
 
 
+def s(x):
+    return can_compile(x).body[0].value.s
+
+
 def test_ast_bad_type():
     "Make sure AST breakage can happen"
     class C:
@@ -480,12 +485,31 @@ def test_ast_unicode_strings():
 
 
 def test_ast_unicode_vs_bytes():
-    def f(x): return can_compile(x).body[0].value.s
-    assert f('"hello"') == u"hello"
-    assert type(f('"hello"')) is (str if PY3 else unicode)  # noqa
-    assert f('b"hello"') == (eval('b"hello"') if PY3 else "hello")
-    assert type(f('b"hello"')) == (bytes if PY3 else str)
-    assert f('b"\\xa0"') == (bytes([160]) if PY3 else chr(160))
+    assert s('"hello"') == u"hello"
+    assert type(s('"hello"')) is (str if PY3 else unicode)  # noqa
+    assert s('b"hello"') == (eval('b"hello"') if PY3 else "hello")
+    assert type(s('b"hello"')) is (bytes if PY3 else str)
+    assert s('b"\\xa0"') == (bytes([160]) if PY3 else chr(160))
+
+
+def test_ast_bracket_string():
+    assert s(r'#[[empty delims]]') == 'empty delims'
+    assert s(r'#[my delim[fizzle]my delim]') == 'fizzle'
+    assert s(r'#[[]]') == ''
+    assert s(r'#[my delim[]my delim]') == ''
+    assert type(s('#[X[hello]X]')) is (str if PY3 else unicode)  # noqa
+    assert s(r'#[X[raw\nstring]X]') == 'raw\\nstring'
+    assert s(r'#[foozle[aa foozli bb ]foozle]') == 'aa foozli bb '
+    assert s(r'#[([unbalanced](]') == 'unbalanced'
+    assert s(r'#[(1💯@)} {a![hello world](1💯@)} {a!]') == 'hello world'
+    assert (s(r'''#[X[
+Remove the leading newline, please.
+]X]''') == 'Remove the leading newline, please.\n')
+    assert (s(r'''#[X[
+
+
+Only one leading newline should be removed.
+]X]''') == '\n\nOnly one leading newline should be removed.\n')
 
 
 def test_compile_error():

tests/test_lex.py

@@ -69,6 +69,17 @@ bc"
     assert objs == [HyString("abc")]
 
 
+def test_lex_bracket_strings():
+
+    objs = tokenize("#[my delim[hello world]my delim]")
+    assert objs == [HyString("hello world")]
+    assert objs[0].brackets == "my delim"
+
+    objs = tokenize("#[[squid]]")
+    assert objs == [HyString("squid")]
+    assert objs[0].brackets == ""
+
+
 def test_lex_integers():
     """ Make sure that integers are valid expressions"""
     objs = tokenize("42 ")