From 564c030950fb7fc4d4380a927daac3601bda473a Mon Sep 17 00:00:00 2001 From: Yigong Wang Date: Wed, 4 Apr 2018 20:37:07 -0400 Subject: [PATCH] Raise LexException when codec can't decode some bytes --- NEWS.rst | 1 + hy/lex/parser.py | 6 +++++- tests/test_lex.py | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 709bd50..232e986 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -31,6 +31,7 @@ Bug Fixes * Fix `(return)` so it works correctly to exit a Python 2 generator * Fixed a case where `->` and `->>` duplicated an argument * Fixed bugs that caused `defclass` to drop statements or crash +* Fixed a REPL crash caused by illegle unicode escape string inputs Misc. Improvements ---------------------------- diff --git a/hy/lex/parser.py b/hy/lex/parser.py index d4814f5..1f5b2d8 100755 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -309,7 +309,11 @@ def t_empty_list(p): def t_string(p): # Replace the single double quotes with triple double quotes to allow # embedded newlines. - s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""') + try: + s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""') + except SyntaxError: + raise LexException("Can't convert {} to a HyString".format(p[0].value), + p[0].source_pos.lineno, p[0].source_pos.colno) return (HyString if isinstance(s, str_type) else HyBytes)(s) diff --git a/tests/test_lex.py b/tests/test_lex.py index a071181..3fb94fd 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -69,6 +69,13 @@ bc" assert objs == [HyString("abc")] +def test_lex_strings_exception(): + """ Make sure tokenize throws when codec can't decode some bytes""" + with lexe() as execinfo: + tokenize('\"\\x8\"') + assert "Can't convert \"\\x8\" to a HyString" in str(execinfo.value) + + def test_lex_bracket_strings(): objs = tokenize("#[my delim[hello world]my delim]")