updating tests

hy/lex/states.py

@@ -4,6 +4,9 @@ from hy.lex.machine import Machine
 from hy.lang.list import HYList
 
 
+WHITESPACE = [" ", "\t", "\n", "\r"]
+
+
 class State(object):
     def __init__(self, machine):
         self.machine = machine
@@ -38,12 +41,9 @@ class Comment(State):
 
 class Idle(State):
     def p(self, x):
-        if x == ";":
+        if x == ";": return Comment
-            return Comment
+        if x == "(": return Expression
-        if x == "(":
+        if x in WHITESPACE: return
-            return Expression
-        if x in [" ", "\t", "\n", "\r"]:
-            return
 
         raise LexException("Unknown char: %s" % (x))
 
@@ -65,20 +65,12 @@ class Expression(State):
             self.bulk = ""
 
     def p(self, x):
-        if x == ")":
+        if x == ")": return Idle
-            return Idle
+        if x in WHITESPACE: self.commit(); return
-        if x == " ":
+        if x == "\"": self.sub(String); return
-            self.commit()
+        if x == "(": self.sub(Expression); return
-            return
+        if x == "[": self.sub(List); return
-        if x == "\"":
+        if x == ";": self.sub(Comment); return
-            self.sub(String)
-            return
-        if x == "(":
-            self.sub(Expression)
-            return
-        if x == "[":
-            self.sub(List)
-            return
         self.bulk += x
 
 
@@ -98,30 +90,17 @@ class List(State):
             self.bulk = ""
 
     def p(self, x):
-        if x == "]":
+        if x == "]": return Idle
-            return Idle
+        if x in WHITESPACE: self.commit(); return
-        if x == " ":
+        if x == "\"": self.sub(String); return
-            self.commit()
+        if x == "[": self.sub(List); return
-            return
+        if x == "(": self.sub(Expression); return
-        if x == "\"":
+        if x == ";": self.sub(Comment); return
-            self.sub(String)
-            return
-        if x == "[":
-            self.sub(List)
-            return
-        if x == "(":
-            self.sub(Expression)
-            return
         self.bulk += x
 
 
 class String(State):
-    magic = {
+    magic = { "n": "\n", "t": "\t", "\\": "\\", "\"": "\"" }
-        "n": "\n",
-        "t": "\t",
-        "\\": "\\",
-        "\"": "\""
-    }
 
     def enter(self):
         self.buf = ""
@@ -140,6 +119,7 @@ class String(State):
 
         if self.esc and x not in self.magic:
             raise LexException("Unknown escape: \\%s" % (x))
+
         elif self.esc:
             x = self.magic[x]
 

tests/lexer/test_basic_lexing.py

@@ -38,6 +38,21 @@ def test_mid_recurse():
     ] == tokenize("(fn one (fn two)(fn three))")
 
 
+def test_mid_recurse_comment():
+    """ Test some crazy recursion with a comment """
+
+    assert [
+        ['fn',
+            'one',
+            ['fn', 'two'],
+            ['fn', 'three'],
+        ]
+    ] == tokenize("""
+(fn one ; this is a test
+    (fn two)(fn three)) ; and so is this
+""")
+
+
 def test_full_recurse():
     """ Test something we could see for real """
     assert [

tests/lexer/test_list_lexing.py

@@ -23,3 +23,22 @@ def test_double_rainbow():
     assert fn == [
         "fn", ["1", "2", "3", "4"], ["5", "6", "7"]
     ]
+
+
+def test_string_in_list():
+    """ String in list """
+    fn = tokenize('(fn [1 2 "three four" 5])')[0]
+    assert fn == [
+        "fn", ["1", "2", "three four", "5"]
+    ]
+
+
+def test_list_recurse_with_comment():
+    """ test we can recurse lists """
+    fn = tokenize("""
+(fn [1 ; this is a test
+     2 3 4 [5 6 7]])
+""")[0]
+    assert fn == [
+        "fn", ["1", "2", "3", "4", ["5", "6", "7"]]
+    ]