Raise LexException when codec can't decode some bytes

This commit is contained in:
Yigong Wang 2018-04-04 20:37:07 -04:00 committed by Kodi Arfer
parent 4d98cde663
commit 564c030950
3 changed files with 13 additions and 1 deletions

View File

@ -31,6 +31,7 @@ Bug Fixes
* Fix `(return)` so it works correctly to exit a Python 2 generator
* Fixed a case where `->` and `->>` duplicated an argument
* Fixed bugs that caused `defclass` to drop statements or crash
* Fixed a REPL crash caused by illegle unicode escape string inputs
Misc. Improvements
----------------------------

View File

@ -309,7 +309,11 @@ def t_empty_list(p):
def t_string(p):
# Replace the single double quotes with triple double quotes to allow
# embedded newlines.
s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
try:
s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
except SyntaxError:
raise LexException("Can't convert {} to a HyString".format(p[0].value),
p[0].source_pos.lineno, p[0].source_pos.colno)
return (HyString if isinstance(s, str_type) else HyBytes)(s)

View File

@ -69,6 +69,13 @@ bc"
assert objs == [HyString("abc")]
def test_lex_strings_exception():
""" Make sure tokenize throws when codec can't decode some bytes"""
with lexe() as execinfo:
tokenize('\"\\x8\"')
assert "Can't convert \"\\x8\" to a HyString" in str(execinfo.value)
def test_lex_bracket_strings():
objs = tokenize("#[my delim[hello world]my delim]")