From 0a633d802465cf7a8fffe48e00d70e81413656cd Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 8 Sep 2017 11:27:11 -0700 Subject: [PATCH 1/4] Correct a ReST syntax typo --- docs/language/api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/language/api.rst b/docs/language/api.rst index b1a3e1d..b051565 100644 --- a/docs/language/api.rst +++ b/docs/language/api.rst @@ -52,8 +52,8 @@ digits. (print 10,000,000,000 10_000_000_000) -Unlike Python, Hy provides literal forms for NaN and infinity: `NaN`, `Inf`, -and `-Inf`. +Unlike Python, Hy provides literal forms for NaN and infinity: ``NaN``, +``Inf``, and ``-Inf``. string literals --------------- From eb23ddc1e2abfc107bbfb23f3db705a0b47993f3 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 8 Sep 2017 11:22:31 -0700 Subject: [PATCH 2/4] =?UTF-8?q?Add=20#[DELIM[=20=E2=80=A6=20]DELIM]=20synt?= =?UTF-8?q?ax=20for=20string=20literals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hy/lex/lexer.py | 6 ++++++ hy/lex/parser.py | 11 +++++++++++ tests/compilers/test_ast.py | 36 ++++++++++++++++++++++++++++++------ tests/test_lex.py | 11 +++++++++++ 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/hy/lex/lexer.py b/hy/lex/lexer.py index cb021a0..b3d1f93 100755 --- a/hy/lex/lexer.py +++ b/hy/lex/lexer.py @@ -27,6 +27,12 @@ lg.add('UNQUOTESPLICE', r'~@%s' % end_quote) lg.add('UNQUOTE', r'~%s' % end_quote) lg.add('DISCARD', r'#_') lg.add('HASHSTARS', r'#\*+') +lg.add('BRACKETSTRING', r'''(?x) + \# \[ ( [^\[\]]* ) \[ # Opening delimiter + \n? # A single leading newline will be ignored + ((?:\n|.)*?) # Content of the string + \] \1 \] # Closing delimiter + ''') lg.add('HASHOTHER', r'#%s' % identifier) # A regexp which matches incomplete strings, used to support diff --git a/hy/lex/parser.py b/hy/lex/parser.py index 4933a92..a289ad7 100755 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -281,6 +281,17 @@ def t_partial_string(p): raise PrematureEndOfInput("Premature end of input") +bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING') +@pg.production("string : BRACKETSTRING") +@set_boundaries +def t_bracket_string(p): + m = bracket_string_re.match(p[0].value) + delim, content = m.groups() + s = HyString(content) + s.brackets = delim + return s + + @pg.production("identifier : IDENTIFIER") @set_boundaries def t_identifier(p): diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index 4e0e00b..b6a5a42 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- # Copyright 2017 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. @@ -46,6 +47,10 @@ def cant_compile(expr): return e +def s(x): + return can_compile(x).body[0].value.s + + def test_ast_bad_type(): "Make sure AST breakage can happen" class C: @@ -480,12 +485,31 @@ def test_ast_unicode_strings(): def test_ast_unicode_vs_bytes(): - def f(x): return can_compile(x).body[0].value.s - assert f('"hello"') == u"hello" - assert type(f('"hello"')) is (str if PY3 else unicode) # noqa - assert f('b"hello"') == (eval('b"hello"') if PY3 else "hello") - assert type(f('b"hello"')) == (bytes if PY3 else str) - assert f('b"\\xa0"') == (bytes([160]) if PY3 else chr(160)) + assert s('"hello"') == u"hello" + assert type(s('"hello"')) is (str if PY3 else unicode) # noqa + assert s('b"hello"') == (eval('b"hello"') if PY3 else "hello") + assert type(s('b"hello"')) is (bytes if PY3 else str) + assert s('b"\\xa0"') == (bytes([160]) if PY3 else chr(160)) + + +def test_ast_bracket_string(): + assert s(r'#[[empty delims]]') == 'empty delims' + assert s(r'#[my delim[fizzle]my delim]') == 'fizzle' + assert s(r'#[[]]') == '' + assert s(r'#[my delim[]my delim]') == '' + assert type(s('#[X[hello]X]')) is (str if PY3 else unicode) # noqa + assert s(r'#[X[raw\nstring]X]') == 'raw\\nstring' + assert s(r'#[foozle[aa foozli bb ]foozle]') == 'aa foozli bb ' + assert s(r'#[([unbalanced](]') == 'unbalanced' + assert s(r'#[(1💯@)} {a![hello world](1💯@)} {a!]') == 'hello world' + assert (s(r'''#[X[ +Remove the leading newline, please. +]X]''') == 'Remove the leading newline, please.\n') + assert (s(r'''#[X[ + + +Only one leading newline should be removed. +]X]''') == '\n\nOnly one leading newline should be removed.\n') def test_compile_error(): diff --git a/tests/test_lex.py b/tests/test_lex.py index 247ae4d..5a21e7f 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -69,6 +69,17 @@ bc" assert objs == [HyString("abc")] +def test_lex_bracket_strings(): + + objs = tokenize("#[my delim[hello world]my delim]") + assert objs == [HyString("hello world")] + assert objs[0].brackets == "my delim" + + objs = tokenize("#[[squid]]") + assert objs == [HyString("squid")] + assert objs[0].brackets == "" + + def test_lex_integers(): """ Make sure that integers are valid expressions""" objs = tokenize("42 ") From deb801edab7b96a9b0d60946fb3d1c8a9a6b3a9f Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 8 Sep 2017 11:23:58 -0700 Subject: [PATCH 3/4] Preserve .brackets in quoted HyStrings --- hy/compiler.py | 6 ++++++ hy/lex/parser.py | 4 +--- hy/models.py | 5 ++++- tests/native_tests/language.hy | 6 ++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/hy/compiler.py b/hy/compiler.py index ca478ed..7ee8811 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -745,6 +745,12 @@ class HyASTCompiler(object): return imports, HyExpression([HySymbol(name), HyString(form)]).replace(form), False + elif isinstance(form, HyString): + x = [HySymbol(name), form] + if form.brackets is not None: + x.extend([HyKeyword(":brackets"), form.brackets]) + return imports, HyExpression(x).replace(form), False + return imports, HyExpression([HySymbol(name), form]).replace(form), False diff --git a/hy/lex/parser.py b/hy/lex/parser.py index a289ad7..05fda85 100755 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -287,9 +287,7 @@ bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING') def t_bracket_string(p): m = bracket_string_re.match(p[0].value) delim, content = m.groups() - s = HyString(content) - s.brackets = delim - return s + return HyString(content, brackets=delim) @pg.production("identifier : IDENTIFIER") diff --git a/hy/models.py b/hy/models.py index 574db49..0d6df4c 100644 --- a/hy/models.py +++ b/hy/models.py @@ -65,7 +65,10 @@ class HyString(HyObject, str_type): scripts. It's either a ``str`` or a ``unicode``, depending on the Python version. """ - pass + def __new__(cls, s=None, brackets=None): + value = super(HyString, cls).__new__(cls, s) + value.brackets = brackets + return value _wrappers[str_type] = HyString diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 1f069af..6a1f126 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -1197,6 +1197,12 @@ (assert (= (eval `(get ~d ~k)) 2))) +(defn test-quote-bracket-string-delim [] + (assert (= (. '#[my delim[hello world]my delim] brackets) "my delim")) + (assert (= (. '#[[squid]] brackets) "")) + (assert (none? (. '"squid" brackets)))) + + (defn test-import-syntax [] "NATIVE: test the import syntax." From 0c229ebda5ae6a32f99e7f858a05f43d51c73867 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Fri, 8 Sep 2017 11:26:59 -0700 Subject: [PATCH 4/4] NEWS and docs for hashstrings --- NEWS | 2 ++ docs/language/api.rst | 41 ++++++++++++++++++++++++++++--------- docs/language/internals.rst | 8 +++++++- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/NEWS b/NEWS index 3e23f66..a4c307a 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,8 @@ Changes from 0.13.0 longer names * Periods are no longer allowed in keywords * `eval` is now a function instead of a special form + * Added a form of string literal called "bracket strings" delimited by + `#[FOO[` and `]FOO]`, where `FOO` is customizable * The compiler now automatically promotes values to Hy model objects as necessary, so you can write ``(eval `(+ 1 ~n))`` instead of ``(eval `(+ 1 ~(HyInteger n)))`` diff --git a/docs/language/api.rst b/docs/language/api.rst index b051565..ed789eb 100644 --- a/docs/language/api.rst +++ b/docs/language/api.rst @@ -58,20 +58,41 @@ Unlike Python, Hy provides literal forms for NaN and infinity: ``NaN``, string literals --------------- -Unlike Python, Hy allows only double-quoted strings (e.g., ``"hello"``). The -single-quote character ``'`` is reserved for preventing the evaluation of a -form (e.g., ``'(+ 1 1)``), as in most Lisps. +Hy allows double-quoted strings (e.g., ``"hello"``), but not single-quoted +strings like Python. The single-quote character ``'`` is reserved for +preventing the evaluation of a form (e.g., ``'(+ 1 1)``), as in most Lisps. Python's so-called triple-quoted strings (e.g., ``'''hello'''`` and ``"""hello"""``) aren't supported. However, in Hy, unlike Python, any string -literal can contain newlines. +literal can contain newlines. Furthermore, Hy supports an alternative form of +string literal called a "bracket string" similar to Lua's long brackets. +Bracket strings have customizable delimiters, like the here-documents of other +languages. A bracket string begins with ``#[FOO[`` and ends with ``]FOO]``, +where ``FOO`` is any string not containing ``[`` or ``]``, including the empty +string. For example:: -Whether running under Python 2 or Python 3, Hy treats string literals as -sequences of Unicode characters by default, and allows you to prefix a literal -with ``b`` to treat it as a sequence of bytes. So when running under Python 3, -Hy translates ``"foo"`` and ``b"foo"`` to the identical Python code, but when -running under Python 2, ``"foo"`` is translated to ``u"foo"`` and ``b"foo"`` is -translated to ``"foo"``. + => (print #[["That's very kind of yuo [sic]" Tom wrote back.]]) + "That's very kind of yuo [sic]" Tom wrote back. + => (print #[==[1 + 1 = 2]==]) + 1 + 1 = 2 + +A bracket string can contain newlines, but if it begins with one, the newline +is removed, so you can begin the content of a bracket string on the line +following the opening delimiter with no effect on the content. Any leading +newlines past the first are preserved. + +Plain string literals support :ref:`a variety of backslash escapes +`. To create a "raw string" that interprets all backslashes +literally, prefix the string with ``r``, as in ``r"slash\not"``. Bracket +strings are always raw strings and don't allow the ``r`` prefix. + +Whether running under Python 2 or Python 3, Hy treats all string literals as +sequences of Unicode characters by default, and allows you to prefix a plain +string literal (but not a bracket string) with ``b`` to treat it as a sequence +of bytes. So when running under Python 3, Hy translates ``"foo"`` and +``b"foo"`` to the identical Python code, but when running under Python 2, +``"foo"`` is translated to ``u"foo"`` and ``b"foo"`` is translated to +``"foo"``. .. _syntax-keywords: diff --git a/docs/language/internals.rst b/docs/language/internals.rst index bf2f4f0..ffd3844 100644 --- a/docs/language/internals.rst +++ b/docs/language/internals.rst @@ -102,7 +102,7 @@ HyString ~~~~~~~~ ``hy.models.HyString`` is the base class of string-equivalent Hy -models. It also represents double-quoted string literals, ``""``, which +models. It also represents string literals (including bracket strings), which compile down to unicode string literals in Python. ``HyStrings`` inherit unicode objects in Python 2, and string objects in Python 3 (and are therefore not encoding-dependent). @@ -113,6 +113,12 @@ Hy literal strings can span multiple lines, and are considered by the parser as a single unit, respecting the Python escapes for unicode strings. +``HyString``\s have an attribute ``brackets`` that stores the custom +delimiter used for a bracket string (e.g., ``"=="`` for ``#[==[hello +world]==]`` and the empty string for ``#[[hello world]]``). +``HyString``\s that are not produced by bracket strings have their +``brackets`` set to ``None``. + HyBytes ~~~~~~~