From f7ff953f7ea382e8f7f8993d772bb215d2037c2b Mon Sep 17 00:00:00 2001
From: "Paul R. Tagliamonte" <paultag@debian.org>
Date: Sat, 2 Mar 2013 21:08:23 -0500
Subject: [PATCH] Updating states / tests for more stuff.

---
 hy/lex/states.py  |  3 +++
 tests/test_lex.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/hy/lex/states.py b/hy/lex/states.py
index 9d67bdd..a2e4fbd 100644
--- a/hy/lex/states.py
+++ b/hy/lex/states.py
@@ -178,4 +178,7 @@ class Idle(State):
         if char == "(":
             return Expression
 
+        if char in WHITESPACE:
+            return
+
         raise LexException("Unknown char (Idle state): `%s`" % (char))
diff --git a/tests/test_lex.py b/tests/test_lex.py
index cee5661..f57612c 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -36,6 +36,12 @@ def test_lex_exception():
     except LexException:
         pass
 
+    try:
+        objs = tokenize("&foo&")
+        assert True == False
+    except LexException:
+        pass
+
 
 def test_lex_expression_symbols():
     """ Make sure that expressions produce symbols """
@@ -71,3 +77,26 @@ def test_lex_line_counting():
 
     assert entry.end_line == 1
     assert entry.end_column == 14
+
+
+def test_lex_line_counting_multi():
+    """ Make sure we can do multi-line tokenization """
+    entries = tokenize("""
+(foo (one two))
+(foo bar)
+""")
+
+    entry = entries[0]
+
+    assert entry.start_line == 2
+    assert entry.start_column == 1
+
+    assert entry.end_line == 2
+    assert entry.end_column == 15
+
+    entry = entries[1]
+    assert entry.start_line == 3
+    assert entry.start_column == 1
+
+    assert entry.end_line == 3
+    assert entry.end_column == 9