diff --git a/hy/lex/states.py b/hy/lex/states.py index 9d67bdd..a2e4fbd 100644 --- a/hy/lex/states.py +++ b/hy/lex/states.py @@ -178,4 +178,7 @@ class Idle(State): if char == "(": return Expression + if char in WHITESPACE: + return + raise LexException("Unknown char (Idle state): `%s`" % (char)) diff --git a/tests/test_lex.py b/tests/test_lex.py index cee5661..f57612c 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -36,6 +36,12 @@ def test_lex_exception(): except LexException: pass + try: + objs = tokenize("&foo&") + assert True == False + except LexException: + pass + def test_lex_expression_symbols(): """ Make sure that expressions produce symbols """ @@ -71,3 +77,26 @@ def test_lex_line_counting(): assert entry.end_line == 1 assert entry.end_column == 14 + + +def test_lex_line_counting_multi(): + """ Make sure we can do multi-line tokenization """ + entries = tokenize(""" +(foo (one two)) +(foo bar) +""") + + entry = entries[0] + + assert entry.start_line == 2 + assert entry.start_column == 1 + + assert entry.end_line == 2 + assert entry.end_column == 15 + + entry = entries[1] + assert entry.start_line == 3 + assert entry.start_column == 1 + + assert entry.end_line == 3 + assert entry.end_column == 9