Merge branch 'master' into pr/271
This commit is contained in:
Paul Tagliamonte
commit
bffe3a05fa
@ -19,8 +19,7 @@
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
__appname__ = "hy"
|
||||
__version__ = "0.9.10"
|
||||
from hy.version import __version__, __appname__ # NOQA
|
||||
|
||||
|
||||
from hy.models.expression import HyExpression # NOQA
|
||||
|
||||
@ -31,18 +31,16 @@ import sys
|
||||
|
||||
import hy
|
||||
|
||||
from hy.importer import ast_compile, import_buffer_to_module
|
||||
from hy.lex.states import Idle, LexException
|
||||
from hy.lex.machine import Machine
|
||||
from hy.lex import LexException, PrematureEndOfInput, tokenize
|
||||
from hy.compiler import hy_compile
|
||||
from hy.importer import ast_compile, import_buffer_to_module
|
||||
from hy.completer import completion
|
||||
|
||||
from hy.macros import macro, require, process
|
||||
from hy.macros import macro, require
|
||||
from hy.models.expression import HyExpression
|
||||
from hy.models.string import HyString
|
||||
from hy.models.symbol import HySymbol
|
||||
|
||||
_machine = Machine(Idle, 1, 0)
|
||||
|
||||
try:
|
||||
import __builtin__ as builtins
|
||||
@ -72,27 +70,14 @@ builtins.exit = HyQuitter('exit')
|
||||
|
||||
class HyREPL(code.InteractiveConsole):
|
||||
def runsource(self, source, filename='<input>', symbol='single'):
|
||||
global _machine
|
||||
|
||||
try:
|
||||
_machine.process(source + "\n")
|
||||
tokens = tokenize(source)
|
||||
except PrematureEndOfInput:
|
||||
return True
|
||||
except LexException:
|
||||
_machine = Machine(Idle, 1, 0)
|
||||
self.showsyntaxerror(filename)
|
||||
return False
|
||||
|
||||
if type(_machine.state) != Idle:
|
||||
_machine = Machine(Idle, 1, 0)
|
||||
return True
|
||||
|
||||
try:
|
||||
tokens = process(_machine.nodes, "__console__")
|
||||
except Exception:
|
||||
_machine = Machine(Idle, 1, 0)
|
||||
self.showtraceback()
|
||||
return False
|
||||
|
||||
_machine = Machine(Idle, 1, 0)
|
||||
try:
|
||||
_ast = hy_compile(tokens, "__console__", root=ast.Interactive)
|
||||
code = ast_compile(_ast, filename, symbol)
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
# Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
|
||||
# Copyright (c) 2013 Julien Danjou <julien@danjou.info>
|
||||
# Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
|
||||
# Copyright (c) 2013 James King <james@agentultra.com>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
|
||||
@ -18,16 +18,21 @@
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
from hy.lex.machine import Machine
|
||||
from hy.lex.states import Idle, LexException
|
||||
from rply.errors import LexingError
|
||||
|
||||
from hy.lex.exceptions import LexException, PrematureEndOfInput # NOQA
|
||||
from hy.lex.lexer import lexer
|
||||
from hy.lex.parser import parser
|
||||
|
||||
|
||||
def tokenize(buf):
|
||||
"""
|
||||
Tokenize a Lisp file or string buffer into internal Hy objects.
|
||||
"""
|
||||
machine = Machine(Idle, 1, 0)
|
||||
machine.process(buf)
|
||||
if type(machine.state) != Idle:
|
||||
raise LexException("Incomplete Lex.")
|
||||
return machine.nodes
|
||||
try:
|
||||
return parser.parse(lexer.lex(buf))
|
||||
except LexingError as e:
|
||||
pos = e.getsourcepos()
|
||||
raise LexException(
|
||||
"Could not identify the next token at line %s, column %s" % (
|
||||
pos.lineno, pos.colno))
|
||||
|
||||
31
hy/lex/exceptions.py
Normal file
31
hy/lex/exceptions.py
Normal file
@ -0,0 +1,31 @@
|
||||
# Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
from hy.errors import HyError
|
||||
|
||||
|
||||
class LexException(HyError):
|
||||
"""Error during the Lexing of a Hython expression."""
|
||||
pass
|
||||
|
||||
|
||||
class PrematureEndOfInput(LexException):
|
||||
"""We got a premature end of input"""
|
||||
pass
|
||||
66
hy/lex/lexer.py
Normal file
66
hy/lex/lexer.py
Normal file
@ -0,0 +1,66 @@
|
||||
# Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
from rply import LexerGenerator
|
||||
|
||||
|
||||
lg = LexerGenerator()
|
||||
|
||||
|
||||
# A regexp for something that should end a quoting/unquoting operator
|
||||
# i.e. a space or a closing brace/paren/curly
|
||||
end_quote = r'(?![\s\)\]\}])'
|
||||
|
||||
|
||||
lg.add('LPAREN', r'\(')
|
||||
lg.add('RPAREN', r'\)')
|
||||
lg.add('LBRACKET', r'\[')
|
||||
lg.add('RBRACKET', r'\]')
|
||||
lg.add('LCURLY', r'\{')
|
||||
lg.add('RCURLY', r'\}')
|
||||
lg.add('QUOTE', r'\'%s' % end_quote)
|
||||
lg.add('QUASIQUOTE', r'`%s' % end_quote)
|
||||
lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
|
||||
lg.add('UNQUOTE', r'~%s' % end_quote)
|
||||
lg.add('HASHBANG', r'#!.*[^\r\n]')
|
||||
|
||||
|
||||
lg.add('STRING', r'''(?x)
|
||||
(?:u|r|ur|ru)? # prefix
|
||||
" # start string
|
||||
(?:
|
||||
| [^"\\] # non-quote or backslash
|
||||
| \\. # or escaped single character
|
||||
| \\x[0-9a-fA-F]{2} # or escaped raw character
|
||||
| \\u[0-9a-fA-F]{4} # or unicode escape
|
||||
| \\U[0-9a-fA-F]{8} # or long unicode escape
|
||||
)* # one or more times
|
||||
" # end string
|
||||
''')
|
||||
|
||||
|
||||
lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+')
|
||||
|
||||
|
||||
lg.ignore(r';.*[\r\n]+')
|
||||
lg.ignore(r'\s+')
|
||||
|
||||
|
||||
lexer = lg.build()
|
||||
@ -1,101 +0,0 @@
|
||||
# Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
from hy.lex.states import Idle, LexException
|
||||
|
||||
|
||||
class Machine(object):
|
||||
"""
|
||||
Hy State Machine. This controls all the state hopping we need to do
|
||||
to properly parse Hy source.
|
||||
"""
|
||||
|
||||
__slots__ = ("submachine", "nodes", "state", "line", "column",
|
||||
"start_line", "start_column")
|
||||
|
||||
def __init__(self, state, line, column):
|
||||
self.nodes = []
|
||||
self.line = line
|
||||
self.column = column
|
||||
self.submachine = None
|
||||
self.state = None
|
||||
self.set_state(state)
|
||||
|
||||
def set_state(self, state):
|
||||
"""
|
||||
Set the new internal machine state. This helps keep line annotations
|
||||
correct, and make sure that we properly call enter and exit.
|
||||
"""
|
||||
|
||||
if self.state:
|
||||
self.state._exit()
|
||||
|
||||
self.accept_result(self.state)
|
||||
|
||||
self.state = state(self)
|
||||
self.state._enter()
|
||||
|
||||
self.start_line = self.line
|
||||
self.start_column = self.column
|
||||
|
||||
def sub(self, state):
|
||||
"""
|
||||
Set up a submachine for this machine.
|
||||
"""
|
||||
self.submachine = Machine(state, self.line, self.column)
|
||||
|
||||
def accept_result(self, state):
|
||||
"""
|
||||
Accept and annotate the result.
|
||||
"""
|
||||
if state and not state.result is None:
|
||||
result = state.result
|
||||
|
||||
result.start_line, result.end_line = (self.start_line, self.line)
|
||||
result.start_column, result.end_column = (self.start_column,
|
||||
self.column)
|
||||
self.nodes.append(result)
|
||||
|
||||
def process(self, buf):
|
||||
"""
|
||||
process an iterable of chars into Hy internal models of the Source.
|
||||
"""
|
||||
for char in buf:
|
||||
|
||||
self.column += 1
|
||||
if char == "\n":
|
||||
self.line += 1
|
||||
self.column = 0
|
||||
|
||||
if self.submachine:
|
||||
self.submachine.process([char])
|
||||
if type(self.submachine.state) == Idle:
|
||||
if len(self.submachine.nodes) > 1:
|
||||
raise LexException("Funky Submachine stuff")
|
||||
|
||||
nodes = self.submachine.nodes
|
||||
self.submachine = None
|
||||
if nodes != []:
|
||||
self.state.nodes.append(nodes[0])
|
||||
continue
|
||||
|
||||
new = self.state.process(char)
|
||||
if new:
|
||||
self.set_state(new)
|
||||
256
hy/lex/parser.py
Normal file
256
hy/lex/parser.py
Normal file
@ -0,0 +1,256 @@
|
||||
# Copyright (c) 2013 Nicolas Dandrimont <nicolas.dandrimont@crans.org>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
from functools import wraps
|
||||
|
||||
from rply import ParserGenerator
|
||||
|
||||
from hy.models.complex import HyComplex
|
||||
from hy.models.dict import HyDict
|
||||
from hy.models.expression import HyExpression
|
||||
from hy.models.float import HyFloat
|
||||
from hy.models.integer import HyInteger
|
||||
from hy.models.keyword import HyKeyword
|
||||
from hy.models.lambdalist import HyLambdaListKeyword
|
||||
from hy.models.list import HyList
|
||||
from hy.models.string import HyString
|
||||
from hy.models.symbol import HySymbol
|
||||
|
||||
from .lexer import lexer
|
||||
from .exceptions import LexException, PrematureEndOfInput
|
||||
|
||||
|
||||
pg = ParserGenerator(
|
||||
[rule.name for rule in lexer.rules] + ['$end'],
|
||||
cache_id="hy_parser"
|
||||
)
|
||||
|
||||
|
||||
def set_boundaries(fun):
|
||||
@wraps(fun)
|
||||
def wrapped(p):
|
||||
start = p[0].source_pos
|
||||
end = p[-1].source_pos
|
||||
ret = fun(p)
|
||||
ret.start_line = start.lineno
|
||||
ret.start_column = start.colno
|
||||
if start is not end:
|
||||
ret.end_line = end.lineno
|
||||
ret.end_column = end.colno
|
||||
else:
|
||||
ret.end_line = start.lineno
|
||||
ret.end_column = start.colno + len(p[0].value)
|
||||
return ret
|
||||
return wrapped
|
||||
|
||||
|
||||
def set_quote_boundaries(fun):
|
||||
@wraps(fun)
|
||||
def wrapped(p):
|
||||
start = p[0].source_pos
|
||||
ret = fun(p)
|
||||
ret.start_line = start.lineno
|
||||
ret.start_column = start.colno
|
||||
ret.end_line = p[-1].end_line
|
||||
ret.end_column = p[-1].end_column
|
||||
return ret
|
||||
return wrapped
|
||||
|
||||
|
||||
@pg.production("main : HASHBANG real_main")
|
||||
def main_hashbang(p):
|
||||
return p[1]
|
||||
|
||||
|
||||
@pg.production("main : real_main")
|
||||
def main(p):
|
||||
return p[0]
|
||||
|
||||
|
||||
@pg.production("real_main : list_contents")
|
||||
def real_main(p):
|
||||
return p[0]
|
||||
|
||||
|
||||
@pg.production("real_main : $end")
|
||||
def real_main_empty(p):
|
||||
return []
|
||||
|
||||
|
||||
@pg.production("paren : LPAREN list_contents RPAREN")
|
||||
@set_boundaries
|
||||
def paren(p):
|
||||
return HyExpression(p[1])
|
||||
|
||||
|
||||
@pg.production("paren : LPAREN RPAREN")
|
||||
@set_boundaries
|
||||
def empty_paren(p):
|
||||
return HyExpression([])
|
||||
|
||||
|
||||
@pg.production("list_contents : term list_contents")
|
||||
def list_contents(p):
|
||||
return [p[0]] + p[1]
|
||||
|
||||
|
||||
@pg.production("list_contents : term")
|
||||
def list_contents_single(p):
|
||||
return [p[0]]
|
||||
|
||||
|
||||
@pg.production("term : identifier")
|
||||
@pg.production("term : paren")
|
||||
@pg.production("term : dict")
|
||||
@pg.production("term : list")
|
||||
@pg.production("term : string")
|
||||
def term(p):
|
||||
return p[0]
|
||||
|
||||
|
||||
@pg.production("term : QUOTE term")
|
||||
@set_quote_boundaries
|
||||
def term_quote(p):
|
||||
return HyExpression([HySymbol("quote"), p[1]])
|
||||
|
||||
|
||||
@pg.production("term : QUASIQUOTE term")
|
||||
@set_quote_boundaries
|
||||
def term_quasiquote(p):
|
||||
return HyExpression([HySymbol("quasiquote"), p[1]])
|
||||
|
||||
|
||||
@pg.production("term : UNQUOTE term")
|
||||
@set_quote_boundaries
|
||||
def term_unquote(p):
|
||||
return HyExpression([HySymbol("unquote"), p[1]])
|
||||
|
||||
|
||||
@pg.production("term : UNQUOTESPLICE term")
|
||||
@set_quote_boundaries
|
||||
def term_unquote_splice(p):
|
||||
return HyExpression([HySymbol("unquote_splice"), p[1]])
|
||||
|
||||
|
||||
@pg.production("dict : LCURLY list_contents RCURLY")
|
||||
@set_boundaries
|
||||
def t_dict(p):
|
||||
return HyDict(p[1])
|
||||
|
||||
|
||||
@pg.production("dict : LCURLY RCURLY")
|
||||
@set_boundaries
|
||||
def empty_dict(p):
|
||||
return HyDict([])
|
||||
|
||||
|
||||
@pg.production("list : LBRACKET list_contents RBRACKET")
|
||||
@set_boundaries
|
||||
def t_list(p):
|
||||
return HyList(p[1])
|
||||
|
||||
|
||||
@pg.production("list : LBRACKET RBRACKET")
|
||||
@set_boundaries
|
||||
def t_empty_list(p):
|
||||
return HyList([])
|
||||
|
||||
|
||||
if sys.version_info[0] >= 3:
|
||||
def uni_hystring(s):
|
||||
return HyString(eval(s))
|
||||
else:
|
||||
def uni_hystring(s):
|
||||
return HyString(eval('u'+s))
|
||||
|
||||
|
||||
@pg.production("string : STRING")
|
||||
@set_boundaries
|
||||
def t_string(p):
|
||||
# remove trailing quote
|
||||
s = p[0].value[:-1]
|
||||
# get the header
|
||||
header, s = s.split('"', 1)
|
||||
# remove unicode marker
|
||||
header = header.replace("u", "")
|
||||
# build python string
|
||||
s = header + '"""' + s + '"""'
|
||||
return uni_hystring(s)
|
||||
|
||||
|
||||
@pg.production("identifier : IDENTIFIER")
|
||||
@set_boundaries
|
||||
def t_identifier(p):
|
||||
obj = p[0].value
|
||||
|
||||
try:
|
||||
return HyInteger(obj)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
return HyFloat(obj)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if obj != 'j':
|
||||
try:
|
||||
return HyComplex(obj)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
table = {
|
||||
"true": "True",
|
||||
"false": "False",
|
||||
"null": "None",
|
||||
}
|
||||
|
||||
if obj in table:
|
||||
return HySymbol(table[obj])
|
||||
|
||||
if obj.startswith(":"):
|
||||
return HyKeyword(obj)
|
||||
|
||||
if obj.startswith("&"):
|
||||
return HyLambdaListKeyword(obj)
|
||||
|
||||
if obj.startswith("*") and obj.endswith("*") and obj not in ("*", "**"):
|
||||
obj = obj[1:-1].upper()
|
||||
|
||||
if "-" in obj and obj != "-":
|
||||
obj = obj.replace("-", "_")
|
||||
|
||||
return HySymbol(obj)
|
||||
|
||||
|
||||
@pg.error
|
||||
def error_handler(token):
|
||||
tokentype = token.gettokentype()
|
||||
if tokentype == '$end':
|
||||
raise PrematureEndOfInput
|
||||
else:
|
||||
raise LexException(
|
||||
"Ran into a %s where it wasn't expected at line %s, column %s" %
|
||||
(tokentype, token.source_pos.lineno, token.source_pos.colno)
|
||||
)
|
||||
|
||||
|
||||
parser = pg.build()
|
||||
396
hy/lex/states.py
396
hy/lex/states.py
@ -1,396 +0,0 @@
|
||||
# Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
from hy.models.expression import HyExpression
|
||||
from hy.models.integer import HyInteger
|
||||
from hy.models.lambdalist import HyLambdaListKeyword
|
||||
from hy.models.float import HyFloat
|
||||
from hy.models.complex import HyComplex
|
||||
from hy.models.symbol import HySymbol
|
||||
from hy.models.string import HyString
|
||||
from hy.models.keyword import HyKeyword
|
||||
from hy.models.dict import HyDict
|
||||
from hy.models.list import HyList
|
||||
|
||||
from hy.errors import HyError
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
|
||||
WHITESPACE = [" ", "\t", "\n", "\r"]
|
||||
|
||||
|
||||
class LexException(HyError):
|
||||
"""
|
||||
Error during the Lexing of a Hython expression.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
def _resolve_atom(obj):
|
||||
"""
|
||||
Resolve a bare atom into one of the following (in order):
|
||||
|
||||
- Integer
|
||||
- LambdaListKeyword
|
||||
- Float
|
||||
- Complex
|
||||
- Symbol
|
||||
"""
|
||||
try:
|
||||
return HyInteger(obj)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if obj.startswith("&"):
|
||||
return HyLambdaListKeyword(obj)
|
||||
|
||||
try:
|
||||
return HyFloat(obj)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if obj != "j":
|
||||
try:
|
||||
return HyComplex(obj)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
table = {
|
||||
"true": "True",
|
||||
"false": "False",
|
||||
"null": "None",
|
||||
}
|
||||
|
||||
if obj in table:
|
||||
return HySymbol(table[obj])
|
||||
|
||||
if obj.startswith(":"):
|
||||
return HyKeyword(obj)
|
||||
|
||||
if obj.startswith("*") and obj.endswith("*") and obj not in ("*", "**"):
|
||||
obj = obj[1:-1].upper()
|
||||
|
||||
if "-" in obj and obj != "-":
|
||||
obj = obj.replace("-", "_")
|
||||
|
||||
return HySymbol(obj)
|
||||
|
||||
|
||||
class State(object):
|
||||
"""
|
||||
Generic State model.
|
||||
"""
|
||||
|
||||
__slots__ = ("nodes", "machine")
|
||||
__metaclass__ = ABCMeta
|
||||
|
||||
def __init__(self, machine):
|
||||
self.machine = machine
|
||||
|
||||
def _enter(self):
|
||||
""" Internal shim for running global ``enter`` code """
|
||||
self.result = None
|
||||
self.nodes = []
|
||||
self.enter()
|
||||
|
||||
def _exit(self):
|
||||
""" Internal shim for running global ``exit`` code """
|
||||
self.exit()
|
||||
|
||||
def enter(self):
|
||||
"""
|
||||
Overridable ``enter`` routines. Subclasses may implement this.
|
||||
"""
|
||||
pass
|
||||
|
||||
def exit(self):
|
||||
"""
|
||||
Overridable ``exit`` routines. Subclasses may implement this.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def process(self, char):
|
||||
"""
|
||||
Overridable ``process`` routines. Subclasses must implement this to be
|
||||
useful.
|
||||
"""
|
||||
pass # ABC
|
||||
|
||||
|
||||
class ListeyThing(State):
|
||||
|
||||
def enter(self):
|
||||
self.buf = ""
|
||||
|
||||
def commit(self):
|
||||
if self.buf != "":
|
||||
ret = _resolve_atom(self.buf)
|
||||
ret.start_line = self._start_line
|
||||
ret.start_column = self._start_column
|
||||
ret.end_line = self.machine.line
|
||||
ret.end_column = (self.machine.column - 1)
|
||||
|
||||
self.nodes.append(ret)
|
||||
self.buf = ""
|
||||
|
||||
def exit(self):
|
||||
self.commit()
|
||||
self.result = self.result_type(self.nodes)
|
||||
|
||||
def process(self, char):
|
||||
if char == "(":
|
||||
self.commit()
|
||||
self.machine.sub(Expression)
|
||||
return
|
||||
|
||||
if char == "{":
|
||||
self.commit()
|
||||
self.machine.sub(Dict)
|
||||
return
|
||||
|
||||
if char == "[":
|
||||
self.commit()
|
||||
self.machine.sub(List)
|
||||
return
|
||||
|
||||
if char == "\"":
|
||||
self.commit()
|
||||
self.machine.sub(String)
|
||||
return
|
||||
|
||||
if char == ";":
|
||||
self.commit()
|
||||
self.machine.sub(Comment)
|
||||
return
|
||||
|
||||
if char == self.end_char:
|
||||
return Idle
|
||||
|
||||
if char in ")]}":
|
||||
raise LexException("Unexpected closing character: `%s'" % (char))
|
||||
|
||||
if char in WHITESPACE:
|
||||
self.commit()
|
||||
return
|
||||
|
||||
if self.buf == "":
|
||||
self._start_line = self.machine.line
|
||||
self._start_column = self.machine.column
|
||||
|
||||
self.buf += char
|
||||
|
||||
|
||||
class List(ListeyThing):
|
||||
"""
|
||||
This state parses a Hy list (like a Clojure vector) for use in native
|
||||
Python interop.
|
||||
|
||||
[foo 1 2 3 4] is a good example.
|
||||
"""
|
||||
|
||||
result_type = HyList
|
||||
end_char = "]"
|
||||
|
||||
|
||||
class Expression(ListeyThing):
|
||||
"""
|
||||
This state parses a Hy expression (statement, to be evaluated at runtime)
|
||||
for running things & stuff.
|
||||
"""
|
||||
|
||||
result_type = HyExpression
|
||||
end_char = ")"
|
||||
|
||||
|
||||
class Dict(ListeyThing):
|
||||
"""
|
||||
This state parses a Hy dict for things.
|
||||
"""
|
||||
|
||||
def exit(self):
|
||||
self.commit()
|
||||
self.result = HyDict(self.nodes)
|
||||
|
||||
end_char = "}"
|
||||
|
||||
|
||||
class String(State):
|
||||
"""
|
||||
String state. This will handle stuff like:
|
||||
|
||||
(println "foobar")
|
||||
^^^^^^^^ -- String
|
||||
"""
|
||||
|
||||
def enter(self):
|
||||
self.escaped = False
|
||||
|
||||
def exit(self):
|
||||
self.result = HyString("".join(self.nodes))
|
||||
|
||||
def process(self, char):
|
||||
"""
|
||||
State transitions:
|
||||
|
||||
- " - Idle
|
||||
"""
|
||||
if self.escaped:
|
||||
self.escaped = False
|
||||
simple_escapables = tuple('abfnrtv')
|
||||
if char in simple_escapables:
|
||||
self.nodes.append(eval('"\\'+char+'"'))
|
||||
return
|
||||
if char == "\\":
|
||||
self.nodes.append("\\")
|
||||
return
|
||||
if char == "\"":
|
||||
self.nodes.append("\"")
|
||||
return
|
||||
|
||||
raise LexException("Unknown modifier: `%s'" % (char))
|
||||
|
||||
if char == "\"":
|
||||
return Idle
|
||||
|
||||
if char == "\\":
|
||||
self.escaped = True
|
||||
return
|
||||
|
||||
self.nodes.append(char)
|
||||
|
||||
|
||||
class Atom(State):
|
||||
"""
|
||||
This state parses integer constants, boolean constants, and symbols
|
||||
"""
|
||||
|
||||
def __init__(self, machine):
|
||||
State.__init__(self, machine)
|
||||
self.initial_buf = ''
|
||||
|
||||
def enter(self):
|
||||
self.buf = self.initial_buf
|
||||
|
||||
def exit(self):
|
||||
self.result = _resolve_atom(self.buf)
|
||||
|
||||
def process(self, char):
|
||||
"""
|
||||
State transitions:
|
||||
|
||||
- WHITESPACE - Idle
|
||||
- ; - Comment
|
||||
"""
|
||||
|
||||
if char in WHITESPACE:
|
||||
return Idle
|
||||
|
||||
if char == ";":
|
||||
return Comment
|
||||
|
||||
self.buf += char
|
||||
|
||||
|
||||
def AtomStartingWith(initial_char):
|
||||
def AtomFactory(machine):
|
||||
state = Atom(machine)
|
||||
state.initial_buf = initial_char
|
||||
return state
|
||||
return AtomFactory
|
||||
|
||||
|
||||
class Idle(State):
|
||||
"""
|
||||
Idle state. This is the first (and last) thing that we should
|
||||
be in.
|
||||
"""
|
||||
|
||||
def process(self, char):
|
||||
"""
|
||||
State transitions:
|
||||
|
||||
- ( - Expression
|
||||
- [ - List
|
||||
- { - Dict
|
||||
- \" - String
|
||||
- ; - Comment
|
||||
- # - Hash
|
||||
- (default) - Atom
|
||||
"""
|
||||
|
||||
if char == "(":
|
||||
return Expression
|
||||
|
||||
if char == "[":
|
||||
return List
|
||||
|
||||
if char == "{":
|
||||
return Dict
|
||||
|
||||
if char == "\"":
|
||||
return String
|
||||
|
||||
if char == ";":
|
||||
return Comment
|
||||
|
||||
if char == "#":
|
||||
return Hash
|
||||
|
||||
if char in WHITESPACE:
|
||||
return
|
||||
|
||||
return AtomStartingWith(char)
|
||||
|
||||
|
||||
class Comment(State):
|
||||
"""
|
||||
Comment state.
|
||||
"""
|
||||
|
||||
def process(self, char):
|
||||
"""
|
||||
State transitions:
|
||||
|
||||
- \n - Idle
|
||||
- (default) - disregard.
|
||||
"""
|
||||
|
||||
if char == "\n":
|
||||
return Idle
|
||||
|
||||
|
||||
class Hash(State):
|
||||
"""
|
||||
Hash state
|
||||
"""
|
||||
|
||||
def process(self, char):
|
||||
"""
|
||||
State transitions:
|
||||
|
||||
- ! - Comment
|
||||
"""
|
||||
|
||||
if char == "!":
|
||||
return Comment
|
||||
|
||||
raise LexException("Unknown char (Hash state): `%s'" % (char))
|
||||
23
hy/version.py
Normal file
23
hy/version.py
Normal file
@ -0,0 +1,23 @@
|
||||
# Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
__appname__ = "hy"
|
||||
__version__ = "0.9.10"
|
||||
@ -1,3 +1,4 @@
|
||||
-r requirements.txt
|
||||
astor
|
||||
tox
|
||||
nose
|
||||
|
||||
@ -1 +1,2 @@
|
||||
# Nothing, yet! (ish). Check site / dev for more deps!
|
||||
# Check site / dev for more deps!
|
||||
-e git+https://github.com/hylang/rply.git#egg=rply
|
||||
|
||||
23
setup.py
23
setup.py
@ -20,23 +20,39 @@
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
from hy import __appname__, __version__
|
||||
from setuptools import setup
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
PKG = "hy"
|
||||
VERSIONFILE = os.path.join(PKG, "version.py")
|
||||
verstr = "unknown"
|
||||
try:
|
||||
verstrline = open(VERSIONFILE, "rt").read()
|
||||
except EnvironmentError:
|
||||
pass # Okay, there is no version file.
|
||||
else:
|
||||
VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]"
|
||||
mo = re.search(VSRE, verstrline, re.M)
|
||||
if mo:
|
||||
__version__ = mo.group(1)
|
||||
else:
|
||||
raise RuntimeError("if %s.py exists, it is required to be well-formed" % (VERSIONFILE,))
|
||||
|
||||
|
||||
long_description = """Hy is a Python <--> Lisp layer. It helps
|
||||
make things work nicer, and lets Python and the Hy lisp variant play
|
||||
nice together. """
|
||||
|
||||
install_requires = []
|
||||
install_requires = ["rply"]
|
||||
if sys.version_info[0] == 2:
|
||||
install_requires.append('argparse>=1.2.1')
|
||||
if os.name == 'nt':
|
||||
install_requires.append('pyreadline==2.0')
|
||||
|
||||
setup(
|
||||
name=__appname__,
|
||||
name=PKG,
|
||||
version=__version__,
|
||||
install_requires=install_requires,
|
||||
entry_points={
|
||||
@ -55,6 +71,7 @@ setup(
|
||||
package_data={
|
||||
'hy.core': ['*.hy'],
|
||||
},
|
||||
setup_requires=['rply'],
|
||||
author="Paul Tagliamonte",
|
||||
author_email="tag@pault.ag",
|
||||
long_description=long_description,
|
||||
|
||||
@ -27,9 +27,7 @@ from hy.models.symbol import HySymbol
|
||||
from hy.models.string import HyString
|
||||
from hy.models.dict import HyDict
|
||||
|
||||
from hy.lex.states import LexException
|
||||
|
||||
from hy.lex import tokenize
|
||||
from hy.lex import LexException, PrematureEndOfInput, tokenize
|
||||
|
||||
|
||||
def test_lex_exception():
|
||||
@ -37,13 +35,17 @@ def test_lex_exception():
|
||||
try:
|
||||
tokenize("(foo")
|
||||
assert True is False
|
||||
except LexException:
|
||||
except PrematureEndOfInput:
|
||||
pass
|
||||
|
||||
try:
|
||||
tokenize("&foo&")
|
||||
tokenize("{foo bar")
|
||||
assert True is False
|
||||
except LexException:
|
||||
except PrematureEndOfInput:
|
||||
pass
|
||||
try:
|
||||
tokenize("(defn foo [bar]")
|
||||
assert True is False
|
||||
except PrematureEndOfInput:
|
||||
pass
|
||||
|
||||
|
||||
@ -124,6 +126,8 @@ def test_lex_expression_complex():
|
||||
assert objs == [HyExpression([HySymbol("foo"), HyComplex(-0.5j)])]
|
||||
objs = tokenize("(foo 1.e7j)")
|
||||
assert objs == [HyExpression([HySymbol("foo"), HyComplex(1.e7j)])]
|
||||
objs = tokenize("(foo j)")
|
||||
assert objs == [HyExpression([HySymbol("foo"), HySymbol("j")])]
|
||||
|
||||
|
||||
def test_lex_line_counting():
|
||||
@ -222,11 +226,17 @@ def test_escapes():
|
||||
entry = tokenize("(foo \"foo\\n\")")[0]
|
||||
assert entry[1] == "foo\n"
|
||||
|
||||
try:
|
||||
entry = tokenize("(foo \"foo\s\")")[0]
|
||||
assert True is False
|
||||
except LexException:
|
||||
pass
|
||||
entry = tokenize("(foo \"foo\s\")")[0]
|
||||
assert entry[1] == "foo\\s"
|
||||
|
||||
|
||||
def test_unicode_escapes():
|
||||
"""Ensure unicode escapes are handled correctly"""
|
||||
s = r'"a\xac\u1234\u20ac\U00008000"'
|
||||
assert len(s) == 29
|
||||
entry = tokenize(s)[0]
|
||||
assert len(entry) == 5
|
||||
assert [ord(x) for x in entry] == [97, 172, 4660, 8364, 32768]
|
||||
|
||||
|
||||
def test_hashbang():
|
||||
|
||||
@ -63,7 +63,7 @@ def test_bin_hy_cmd():
|
||||
|
||||
ret = run_cmd("hy -c \"(koan\"")
|
||||
assert ret[0] == 1
|
||||
assert "LexException" in ret[1]
|
||||
assert "PrematureEndOfInput" in ret[1]
|
||||
|
||||
|
||||
def test_bin_hy_icmd():
|
||||
|
||||
13
tox.ini
13
tox.ini
@ -5,6 +5,8 @@ commands = nosetests
|
||||
deps =
|
||||
nose
|
||||
setuptools
|
||||
-e
|
||||
git+https://github.com/hylang/rply.git#egg=rply
|
||||
|
||||
[testenv:pypy]
|
||||
commands = nosetests
|
||||
@ -12,6 +14,8 @@ deps =
|
||||
astor
|
||||
nose
|
||||
setuptools
|
||||
-e
|
||||
git+https://github.com/hylang/rply.git#egg=rply
|
||||
|
||||
[testenv:py27]
|
||||
commands = nosetests
|
||||
@ -19,6 +23,8 @@ deps =
|
||||
astor
|
||||
nose
|
||||
setuptools
|
||||
-e
|
||||
git+https://github.com/hylang/rply.git#egg=rply
|
||||
|
||||
[testenv:py26]
|
||||
deps =
|
||||
@ -27,7 +33,12 @@ deps =
|
||||
setuptools
|
||||
unittest2
|
||||
importlib
|
||||
-e
|
||||
git+https://github.com/hylang/rply.git#egg=rply
|
||||
|
||||
[testenv:flake8]
|
||||
deps = flake8
|
||||
deps =
|
||||
flake8
|
||||
-e
|
||||
git+https://github.com/hylang/rply.git#egg=rply
|
||||
commands = flake8 hy bin tests
|
||||
|
||||
Loading…
Reference in New Issue
Block a user