Fiddling with the lexing.

hy/lex/machine.py

@@ -18,7 +18,7 @@ class Machine(object):
         if self.state:
             self.state._exit()
 
-        self.accept_result()
+        self.accept_result(self.state)
 
         self.state = state(self)
         self.state._enter()
@@ -29,18 +29,21 @@ class Machine(object):
     def sub(self, state):
         self.submachine = Machine(state, self.line, self.column)
 
-    def accept_result(self):
+    def accept_result(self, state):
-        if self.state and self.state.result:
+        if state and state.result:
-            self.nodes.append(self.state.result)
+            self.nodes.append(state.result)
 
     def process(self, buf):
         for char in buf:
             if self.submachine:
                 self.submachine.process([char])
                 if type(self.submachine.state) == Idle:
-                    if self.submachine.state.result:
+                    if len(self.submachine.nodes) != 1:
-                        self.state.nodes.append(self.submachine.state.result)
+                        raise LexException("Funky Submachine stuff")
+                    result = self.submachine.nodes[0]
                     self.submachine = None
+                    if result:
+                        self.state.nodes.append(result)
                 continue
 
             new = self.state.process(char)

hy/lex/states.py

@@ -1,5 +1,6 @@
 from hy.models.expression import HyExpression
 from hy.models.symbol import HySymbol
+from hy.models.string import HyString
 from hy.errors import HyError
 
 
@@ -44,7 +45,8 @@ class Expression(State):
         self.buf = ""
 
     def commit(self):
-        self.nodes.append(_resolve_atom(self.buf))
+        if self.buf != "":
+            self.nodes.append(_resolve_atom(self.buf))
         self.buf = ""
 
     def exit(self):
@@ -53,7 +55,12 @@ class Expression(State):
 
     def process(self, char):
         if char == "(":
-            return self.machine.sub(Expression)
+            self.machine.sub(Expression)
+            return
+
+        if char == "\"":
+            self.machine.sub(String)
+            return
 
         if char == ")":
             return Idle
@@ -65,6 +72,17 @@ class Expression(State):
         self.buf += char
 
 
+class String(State):
+    def exit(self):
+        self.result = HyString("".join(self.nodes))
+
+    def process(self, char):
+        if char == "\"":
+            return Idle
+
+        self.nodes.append(char)
+
+
 class Idle(State):
     def process(self, char):
         if char == "(":

tests/test_lex.py

@@ -1,9 +1,13 @@
 from hy.lex import tokenize
 from hy.models.expression import HyExpression
 from hy.models.symbol import HySymbol
+from hy.models.string import HyString
 
 
-def test_lex_expression():
+def test_lex_expression_symbols():
     objs = tokenize("(foo bar)")
-    print objs
     assert objs == [HyExpression([HySymbol("foo"), HySymbol("bar")])]
+
+def test_lex_expression_strings():
+    objs = tokenize("(foo \"bar\")")
+    assert objs == [HyExpression([HySymbol("foo"), HyString("bar")])]