Merge pull request #1572 from waigx/fix/hystring

Raise LexException when codec can't decode some bytes

NEWS.rst

@@ -31,6 +31,7 @@ Bug Fixes
 * Fix `(return)` so it works correctly to exit a Python 2 generator
 * Fixed a case where `->` and `->>` duplicated an argument
 * Fixed bugs that caused `defclass` to drop statements or crash
+* Fixed a REPL crash caused by illegle unicode escape string inputs
 
 Misc. Improvements
 ----------------------------

hy/lex/parser.py

@@ -309,7 +309,11 @@ def t_empty_list(p):
 def t_string(p):
     # Replace the single double quotes with triple double quotes to allow
     # embedded newlines.
-    s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
+    try:
+        s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
+    except SyntaxError:
+        raise LexException("Can't convert {} to a HyString".format(p[0].value),
+            p[0].source_pos.lineno, p[0].source_pos.colno)
     return (HyString if isinstance(s, str_type) else HyBytes)(s)
 
 

tests/test_lex.py

@@ -69,6 +69,13 @@ bc"
     assert objs == [HyString("abc")]
 
 
+def test_lex_strings_exception():
+    """ Make sure tokenize throws when codec can't decode some bytes"""
+    with lexe() as execinfo:
+        tokenize('\"\\x8\"')
+    assert "Can't convert \"\\x8\" to a HyString" in str(execinfo.value)
+
+
 def test_lex_bracket_strings():
 
     objs = tokenize("#[my delim[hello world]my delim]")