Merge pull request #1572 from waigx/fix/hystring

Raise LexException when codec can't decode some bytes
This commit is contained in:
Kodi Arfer 2018-04-07 11:54:52 -07:00 committed by GitHub
commit 2ad3401b36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 13 additions and 1 deletions

View File

@ -31,6 +31,7 @@ Bug Fixes
* Fix `(return)` so it works correctly to exit a Python 2 generator * Fix `(return)` so it works correctly to exit a Python 2 generator
* Fixed a case where `->` and `->>` duplicated an argument * Fixed a case where `->` and `->>` duplicated an argument
* Fixed bugs that caused `defclass` to drop statements or crash * Fixed bugs that caused `defclass` to drop statements or crash
* Fixed a REPL crash caused by illegle unicode escape string inputs
Misc. Improvements Misc. Improvements
---------------------------- ----------------------------

View File

@ -309,7 +309,11 @@ def t_empty_list(p):
def t_string(p): def t_string(p):
# Replace the single double quotes with triple double quotes to allow # Replace the single double quotes with triple double quotes to allow
# embedded newlines. # embedded newlines.
s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""') try:
s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
except SyntaxError:
raise LexException("Can't convert {} to a HyString".format(p[0].value),
p[0].source_pos.lineno, p[0].source_pos.colno)
return (HyString if isinstance(s, str_type) else HyBytes)(s) return (HyString if isinstance(s, str_type) else HyBytes)(s)

View File

@ -69,6 +69,13 @@ bc"
assert objs == [HyString("abc")] assert objs == [HyString("abc")]
def test_lex_strings_exception():
""" Make sure tokenize throws when codec can't decode some bytes"""
with lexe() as execinfo:
tokenize('\"\\x8\"')
assert "Can't convert \"\\x8\" to a HyString" in str(execinfo.value)
def test_lex_bracket_strings(): def test_lex_bracket_strings():
objs = tokenize("#[my delim[hello world]my delim]") objs = tokenize("#[my delim[hello world]my delim]")