Thinking about a refacotr.

example

@@ -0,0 +1,8 @@
+
+# (print ((fn [x] (* x x)) 2))
+
+
+def __hy_anon_fn_00001(x):
+    return x * x
+
+print __hy_anon_fn_00001(2)

hy/__init__.py

@@ -0,0 +1,4 @@
+
+
+__appname__ = "hy"
+__version__ = "0.ALPHA.REALLY.1.0~pre0"

hy/errors.py

@@ -0,0 +1,4 @@
+
+
+class HyError(Exception):
+    pass

hy/lex/__init__.py

@@ -0,0 +1,7 @@
+from hy.lex.machine import Machine
+from hy.lex.states import Idle
+
+def tokenize(buf):
+    machine = Machine(Idle, 0, 0)
+    machine.process(buf)
+    print machine.nodes

hy/lex/machine.py

@@ -0,0 +1,44 @@
+from hy.lex.states import Idle
+
+class Machine(object):
+    __slots__ = ("submachine", "nodes", "state", "line", "column",
+                 "start_line", "start_column")
+
+    def __init__(self, state, line, column):
+        self.nodes = []
+        self.line = line
+        self.column = column
+        self.submachine = None
+        self.state = None
+        self.set_state(state)
+
+
+    def set_state(self, state):
+        if self.state:
+            self.state.exit()
+
+        self.state = state(self)
+        self.state.enter()
+
+        self.start_line = self.line
+        self.start_column = self.column
+
+    def sub(self, state):
+        self.submachine = Machine(state, self.line, self.column)
+
+    def process(self, buf):
+        for char in buf:
+            self.column += 1
+            if char == "\n":
+                self.column = 0
+                self.line += 1
+
+            if self.submachine:
+                self.submachine.process([char])
+                if self.submachine.state == Idle:
+                    self.nodes += self.submachine.nodes
+                    self.submachine = None
+
+            ret = self.state.process(char)
+            if ret:
+                self.set_state(ret)

hy/lex/states.py

@@ -0,0 +1,62 @@
+from hy.errors import HyError
+
+WHITESPACE = [" ", "\t", "\n", "\r"]
+
+
+class LexException(HyError):
+    pass
+
+
+class State(object):
+    __slots__ = ("machine",)
+
+    def __init__(self, machine):
+        self.machine = machine
+
+    def enter(self):
+        pass
+
+    def exit(self):
+        pass
+
+
+class Expression(State):
+    def enter(self):
+        self.nodes = []
+        self.buf = ""
+
+    def commit(self):
+        self.nodes.append(self.buf)
+        self.buf = ""
+
+    def exit(self):
+        self.commit()
+        self.machine.nodes.append(self.nodes)
+
+    def process(self, char):
+        if char == ")":
+            return Idle
+
+        if char == "(":
+            return Expression
+
+        if char in WHITESPACE:
+            self.commit()
+            return
+
+        self.buf += char
+
+
+class Idle(State):
+    def process(self, char):
+        table = {
+            "(": Expression
+        }
+
+        if char in table:
+            return table[char]
+
+        if char in WHITESPACE:
+            return
+
+        raise LexException("Unknown char: %s" % (char))

setup.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+from hy import __appname__, __version__
+from setuptools import setup
+
+long_description = open('README.md', 'r').read()
+
+setup(
+    name=__appname__,
+    version=__version__,
+    packages=[
+        'hy',
+        'hy.lex',
+    ],
+    author="Paul Tagliamonte",
+    author_email="paultag@debian.org",
+    long_description=long_description,
+    description='does some stuff with things & stuff',
+    license="Expat",
+    url="",
+    platforms=['any']
+)

tests/test_lex.py

@@ -0,0 +1,5 @@
+from hy.lex import tokenize
+
+
+def test_lex_expression():
+    objs = tokenize("(foo bar)")