From e536d22a8825668ced7aa420707b157b2b070b9f Mon Sep 17 00:00:00 2001 From: Paul Tagliamonte Date: Sat, 15 Dec 2012 17:07:11 -0500 Subject: [PATCH] Interesting changes. --- hy/lang/__init__.py | 0 hy/lang/expression.py | 3 ++ hy/lang/string.py | 3 ++ hy/lex/errors.py | 2 + hy/lex/machine.py | 16 ++++++++ hy/lex/states.py | 78 +++++++++++++++++++++++++++++++++++ hy/lex/tokenize.py | 95 +------------------------------------------ test.py | 4 +- 8 files changed, 105 insertions(+), 96 deletions(-) create mode 100644 hy/lang/__init__.py create mode 100644 hy/lang/expression.py create mode 100644 hy/lang/string.py create mode 100644 hy/lex/errors.py create mode 100644 hy/lex/machine.py create mode 100644 hy/lex/states.py diff --git a/hy/lang/__init__.py b/hy/lang/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hy/lang/expression.py b/hy/lang/expression.py new file mode 100644 index 0000000..88aa571 --- /dev/null +++ b/hy/lang/expression.py @@ -0,0 +1,3 @@ +class HYExpression(list): + def __init__(self, nodes): + self += nodes diff --git a/hy/lang/string.py b/hy/lang/string.py new file mode 100644 index 0000000..4983da1 --- /dev/null +++ b/hy/lang/string.py @@ -0,0 +1,3 @@ +class HYString(str): + def __init__(self, string): + self = string diff --git a/hy/lex/errors.py b/hy/lex/errors.py new file mode 100644 index 0000000..fc9ae07 --- /dev/null +++ b/hy/lex/errors.py @@ -0,0 +1,2 @@ +class LexException(Exception): + pass diff --git a/hy/lex/machine.py b/hy/lex/machine.py new file mode 100644 index 0000000..d2742d3 --- /dev/null +++ b/hy/lex/machine.py @@ -0,0 +1,16 @@ +class Machine(object): + def __init__(self, state): + # print "New machine: %s" % (state) + self.nodes = [] + self.state = state(self) + self.state.enter() + + def process(self, buf): + for i in range(0, len(buf)): + char = buf[i] + nx = self.state.process(char) + if nx: + # print "New state: %s" % (nx) + self.state.exit() + self.state = nx(self) + self.state.enter() diff --git a/hy/lex/states.py b/hy/lex/states.py new file mode 100644 index 0000000..fe29aa7 --- /dev/null +++ b/hy/lex/states.py @@ -0,0 +1,78 @@ +from hy.lang.expression import HYExpression +from hy.lex.machine import Machine + + +class State(object): + def __init__(self, machine): + self.machine = machine + self.sub_machine = None + + def enter(self): + pass + + def exit(self): + pass + + def process(self, x): + if self.sub_machine: + self.sub_machine.process(x) + idle = type(self.sub_machine.state) == Idle + if idle: + self.nodes.append(self.sub_machine.nodes) + self.sub_machine = None + return + + return self.p(x) + + +class Comment(State): + def p(self, x): + if x == '\n': + return Idle + + +class Idle(State): + def p(self, x): + if x == ";": + return Comment + if x == "(": + return Expression + if x in [" ", "\t", "\n", "\r"]: + return + + raise LexException("Unknown char: %s" % (x)) + + +class Expression(State): + def enter(self): + self.nodes = HYExpression([]) + self.bulk = "" + self.sub_machine = None + + def exit(self): + if self.bulk: + self.nodes.append(self.bulk) + + self.machine.nodes.append(self.nodes) + + def commit(self): + if self.bulk.strip() != "": + self.nodes.append(self.bulk) + self.bulk = "" + + def p(self, x): + if x == ")": + return Idle + + if x == " ": + self.commit() + return + + if x == "\"": + return String + + if x == "(": + self.sub_machine = Machine(Expression) + return + + self.bulk += x diff --git a/hy/lex/tokenize.py b/hy/lex/tokenize.py index 6f50f82..17e99ab 100644 --- a/hy/lex/tokenize.py +++ b/hy/lex/tokenize.py @@ -1,96 +1,5 @@ - -class HYExpression(list): - def __init__(self, nodes): - self += nodes - - -class LexException(Exception): - pass - - -class State(object): - def __init__(self, machine): - self.machine = machine - - def enter(self): pass - def exit(self): pass - def process(self, x): pass - - -class Comment(State): - def process(self, x): - if x == '\n': - return Idle - - -class Idle(State): - def process(self, x): - if x == ";": - return Comment - if x == "(": - return Expression - if x in [" ", "\t", "\n", "\r"]: - return - - raise LexException("Unknown char: %s" % (x)) - - -class Expression(State): - def enter(self): - self.nodes = HYExpression([]) - self.bulk = "" - self.sub_machine = None - - def exit(self): - if self.bulk: - self.nodes.append(self.bulk) - - self.machine.nodes.append(self.nodes) - - def commit(self): - if self.bulk.strip() != "": - self.nodes.append(self.bulk) - self.bulk = "" - - def process(self, x): - if self.sub_machine is not None: - self.sub_machine.process(x) - if type(self.sub_machine.state) == Idle: - self.nodes.append(self.sub_machine.nodes) - self.sub_machine = None - return - - if x == ")": - return Idle - - if x == " ": - self.commit() - return - - if x == "(": - self.sub_machine = Machine(Expression) - return - - self.bulk += x - - -class Machine(object): - def __init__(self, state): - # print "New machine: %s" % (state) - self.nodes = [] - self.state = state(self) - self.state.enter() - - def process(self, buf): - for i in range(0, len(buf)): - char = buf[i] - nx = self.state.process(char) - if nx: - # print "New state: %s" % (nx) - self.state.exit() - self.state = nx(self) - self.state.enter() - +from hy.lex.machine import Machine +from hy.lex.states import Idle def tokenize(buff): m = Machine(Idle) diff --git a/test.py b/test.py index 9cdac1e..f75a744 100644 --- a/test.py +++ b/test.py @@ -2,7 +2,5 @@ from hy.lex.tokenize import tokenize print tokenize(""" -(+ 1 1) ; this adds one plus one -(- 1 1) ; this does other things -(print (+ 1 1)) +(+ 2 (+ 1 1) (- 1 1)) """)