Interesting changes.

hy/lang/expression.py

@@ -0,0 +1,3 @@
+class HYExpression(list):
+    def __init__(self, nodes):
+        self += nodes

hy/lang/string.py

@@ -0,0 +1,3 @@
+class HYString(str):
+    def __init__(self, string):
+        self = string

hy/lex/errors.py

@@ -0,0 +1,2 @@
+class LexException(Exception):
+    pass

hy/lex/machine.py

@@ -0,0 +1,16 @@
+class Machine(object):
+    def __init__(self, state):
+        # print "New machine: %s" % (state)
+        self.nodes = []
+        self.state = state(self)
+        self.state.enter()
+
+    def process(self, buf):
+        for i in range(0, len(buf)):
+            char = buf[i]
+            nx = self.state.process(char)
+            if nx:
+                # print "New state: %s" % (nx)
+                self.state.exit()
+                self.state = nx(self)
+                self.state.enter()

hy/lex/states.py

@@ -0,0 +1,78 @@
+from hy.lang.expression import HYExpression
+from hy.lex.machine import Machine
+
+
+class State(object):
+    def __init__(self, machine):
+        self.machine = machine
+        self.sub_machine = None
+
+    def enter(self):
+        pass
+
+    def exit(self):
+        pass
+
+    def process(self, x):
+        if self.sub_machine:
+            self.sub_machine.process(x)
+            idle = type(self.sub_machine.state) == Idle
+            if idle:
+                self.nodes.append(self.sub_machine.nodes)
+                self.sub_machine = None
+            return
+
+        return self.p(x)
+
+
+class Comment(State):
+    def p(self, x):
+        if x == '\n':
+            return Idle
+
+
+class Idle(State):
+    def p(self, x):
+        if x == ";":
+            return Comment
+        if x == "(":
+            return Expression
+        if x in [" ", "\t", "\n", "\r"]:
+            return
+
+        raise LexException("Unknown char: %s" % (x))
+
+
+class Expression(State):
+    def enter(self):
+        self.nodes = HYExpression([])
+        self.bulk = ""
+        self.sub_machine = None
+
+    def exit(self):
+        if self.bulk:
+            self.nodes.append(self.bulk)
+
+        self.machine.nodes.append(self.nodes)
+
+    def commit(self):
+        if self.bulk.strip() != "":
+            self.nodes.append(self.bulk)
+            self.bulk = ""
+
+    def p(self, x):
+        if x == ")":
+            return Idle
+
+        if x == " ":
+            self.commit()
+            return
+
+        if x == "\"":
+            return String
+
+        if x == "(":
+            self.sub_machine = Machine(Expression)
+            return
+
+        self.bulk += x

hy/lex/tokenize.py

@@ -1,96 +1,5 @@
-
-class HYExpression(list):
-    def __init__(self, nodes):
-        self += nodes
-
-
-class LexException(Exception):
-    pass
-
-
-class State(object):
-    def __init__(self, machine):
-        self.machine = machine
-
-    def enter(self): pass
-    def exit(self): pass
-    def process(self, x): pass
-
-
-class Comment(State):
-    def process(self, x):
-        if x == '\n':
-            return Idle
-
-
-class Idle(State):
-    def process(self, x):
-        if x == ";":
-            return Comment
-        if x == "(":
-            return Expression
-        if x in [" ", "\t", "\n", "\r"]:
-            return
-
-        raise LexException("Unknown char: %s" % (x))
-
-
-class Expression(State):
-    def enter(self):
-        self.nodes = HYExpression([])
-        self.bulk = ""
-        self.sub_machine = None
-
-    def exit(self):
-        if self.bulk:
-            self.nodes.append(self.bulk)
-
-        self.machine.nodes.append(self.nodes)
-
-    def commit(self):
-        if self.bulk.strip() != "":
-            self.nodes.append(self.bulk)
-            self.bulk = ""
-
-    def process(self, x):
-        if self.sub_machine is not None:
-            self.sub_machine.process(x)
-            if type(self.sub_machine.state) == Idle:
-                self.nodes.append(self.sub_machine.nodes)
-                self.sub_machine = None
-            return
-
-        if x == ")":
-            return Idle
-
-        if x == " ":
-            self.commit()
-            return
-
-        if x == "(":
-            self.sub_machine = Machine(Expression)
-            return
-
-        self.bulk += x
-
-
-class Machine(object):
-    def __init__(self, state):
-        # print "New machine: %s" % (state)
-        self.nodes = []
-        self.state = state(self)
-        self.state.enter()
-
-    def process(self, buf):
-        for i in range(0, len(buf)):
-            char = buf[i]
-            nx = self.state.process(char)
-            if nx:
-                # print "New state: %s" % (nx)
-                self.state.exit()
-                self.state = nx(self)
-                self.state.enter()
-
+from hy.lex.machine import Machine
+from hy.lex.states import Idle
 
 def tokenize(buff):
     m = Machine(Idle)

test.py

@@ -2,7 +2,5 @@
 from hy.lex.tokenize import tokenize
 
 print tokenize("""
-(+ 1 1) ; this adds one plus one
-(- 1 1) ; this does other things
-(print (+ 1 1))
+(+ 2 (+ 1 1) (- 1 1))
 """)