Thinking.

hy/lex/__init__.py

@@ -1,7 +1,9 @@
 from hy.lex.machine import Machine
-from hy.lex.states import Idle
+from hy.lex.states import Idle, LexException
 
 def tokenize(buf):
     machine = Machine(Idle, 0, 0)
     machine.process(buf)
+    if type(machine.state) != Idle:
+        raise LexException("Incomplete Lex.")
     return machine.nodes

hy/lex/machine.py

@@ -1,4 +1,5 @@
-from hy.lex.states import Idle
+from hy.lex.states import Idle, LexException
+
 
 class Machine(object):
     __slots__ = ("submachine", "nodes", "state", "line", "column",
@@ -15,10 +16,12 @@ class Machine(object):
 
     def set_state(self, state):
         if self.state:
-            self.state.exit()
+            self.state._exit()
+
+        self.accept_result()
 
         self.state = state(self)
-        self.state.enter()
+        self.state._enter()
 
         self.start_line = self.line
         self.start_column = self.column
@@ -26,19 +29,20 @@ class Machine(object):
     def sub(self, state):
         self.submachine = Machine(state, self.line, self.column)
 
+    def accept_result(self):
+        if self.state and self.state.result:
+            self.nodes.append(self.state.result)
+
     def process(self, buf):
         for char in buf:
-            self.column += 1
-            if char == "\n":
-                self.column = 0
-                self.line += 1
-
             if self.submachine:
                 self.submachine.process([char])
-                if self.submachine.state == Idle:
-                    self.nodes += self.submachine.nodes
+                if type(self.submachine.state) == Idle:
+                    if self.submachine.state.result:
+                        self.state.nodes.append(self.submachine.state.result)
                     self.submachine = None
+                continue
 
-            ret = self.state.process(char)
-            if ret:
-                self.set_state(ret)
+            new = self.state.process(char)
+            if new:
+                self.set_state(new)

hy/lex/states.py

@@ -15,21 +15,32 @@ def _resolve_atom(obj):
 
 
 class State(object):
-    __slots__ = ("machine",)
+    __slots__ = ("nodes", "machine")
 
     def __init__(self, machine):
         self.machine = machine
 
+    def _enter(self):
+        self.result = None
+        self.nodes = []
+        self.enter()
+
+    def _exit(self):
+        self.exit()
+
     def enter(self):
-        pass
+        pass  # ABC
 
     def exit(self):
-        pass
+        pass  # ABC
+
+    def process(self, char):
+        pass  # ABC
 
 
 class Expression(State):
+
     def enter(self):
-        self.nodes = []
         self.buf = ""
 
     def commit(self):
@@ -38,15 +49,15 @@ class Expression(State):
 
     def exit(self):
         self.commit()
-        self.machine.nodes.append(HyExpression(self.nodes))
+        self.result = HyExpression(self.nodes)
 
     def process(self, char):
+        if char == "(":
+            return self.machine.sub(Expression)
+
         if char == ")":
             return Idle
 
-        if char == "(":
-            return Expression
-
         if char in WHITESPACE:
             self.commit()
             return
@@ -56,14 +67,7 @@ class Expression(State):
 
 class Idle(State):
     def process(self, char):
-        table = {
-            "(": Expression
-        }
+        if char == "(":
+            return Expression
 
-        if char in table:
-            return table[char]
-
-        if char in WHITESPACE:
-            return
-
-        raise LexException("Unknown char: %s" % (char))
+        raise LexException("Unknown char (Idle state): `%s`" % (char))

tests/test_lex.py

@@ -5,4 +5,5 @@ from hy.models.symbol import HySymbol
 
 def test_lex_expression():
     objs = tokenize("(foo bar)")
+    print objs
     assert objs == [HyExpression([HySymbol("foo"), HySymbol("bar")])]