diff --git a/hy/lang/list.py b/hy/lang/list.py new file mode 100644 index 0000000..f0bb973 --- /dev/null +++ b/hy/lang/list.py @@ -0,0 +1,3 @@ +class HYList(list): + def __init__(self, nodes): + self += nodes diff --git a/hy/lex/states.py b/hy/lex/states.py index 6d12468..99131c6 100644 --- a/hy/lex/states.py +++ b/hy/lex/states.py @@ -1,6 +1,7 @@ from hy.lang.expression import HYExpression from hy.lex.errors import LexException from hy.lex.machine import Machine +from hy.lang.list import HYList class State(object): @@ -14,6 +15,9 @@ class State(object): def exit(self): pass + def sub(self, machine): + self.sub_machine = Machine(machine) + def process(self, x): if self.sub_machine: self.sub_machine.process(x) @@ -48,7 +52,6 @@ class Expression(State): def enter(self): self.nodes = HYExpression([]) self.bulk = "" - self.sub_machine = None def exit(self): if self.bulk: @@ -64,19 +67,51 @@ class Expression(State): def p(self, x): if x == ")": return Idle - if x == " ": self.commit() return - if x == "\"": - self.sub_machine = Machine(String) + self.sub(String) return - if x == "(": - self.sub_machine = Machine(Expression) + self.sub(Expression) return + if x == "[": + self.sub(List) + return + self.bulk += x + +class List(State): + def enter(self): + self.nodes = HYList([]) + self.bulk = "" + + def exit(self): + if self.bulk: + self.nodes.append(self.bulk) + self.machine.nodes.append(self.nodes) + + def commit(self): + if self.bulk.strip() != "": + self.nodes.append(self.bulk) + self.bulk = "" + + def p(self, x): + if x == "]": + return Idle + if x == " ": + self.commit() + return + if x == "\"": + self.sub(String) + return + if x == "[": + self.sub(List) + return + if x == "(": + self.sub(Expression) + return self.bulk += x diff --git a/tests/lexer/test_list_lexing.py b/tests/lexer/test_list_lexing.py new file mode 100644 index 0000000..43fef29 --- /dev/null +++ b/tests/lexer/test_list_lexing.py @@ -0,0 +1,8 @@ +from hy.lex.tokenize import tokenize + + +def test_list_lex(): + fn = tokenize("(fn [1 2 3 4])")[0] + assert fn == [ + "fn", ["1", "2", "3", "4"] + ]