Fix .end_line and .end_column of single-token models

This commit is contained in:
Kodi Arfer 2019-02-18 12:00:38 -05:00
parent a338e4c323
commit 997321d31c
3 changed files with 30 additions and 2 deletions

View File

@ -31,8 +31,11 @@ def set_boundaries(fun):
ret.end_line = end.lineno ret.end_line = end.lineno
ret.end_column = end.colno ret.end_column = end.colno
else: else:
ret.end_line = start.lineno v = p[0].value
ret.end_column = start.colno + len(p[0].value) ret.end_line = start.lineno + v.count('\n')
ret.end_column = (len(v) - v.rindex('\n') - 1
if '\n' in v
else start.colno + len(v) - 1)
return ret return ret
return wrapped return wrapped

View File

@ -33,6 +33,11 @@ class HyObject(object):
""" """
Generic Hy Object model. This is helpful to inject things into all the Generic Hy Object model. This is helpful to inject things into all the
Hy lexing Objects at once. Hy lexing Objects at once.
The position properties (`start_line`, `end_line`, `start_column`,
`end_column`) are each 1-based and inclusive. For example, a symbol
`abc` starting at the first column would have `start_column` 1 and
`end_column` 3.
""" """
__properties__ = ["module", "start_line", "end_line", "start_column", __properties__ = ["module", "start_line", "end_line", "start_column",
"end_column"] "end_column"]

View File

@ -247,6 +247,12 @@ def test_lex_column_counting():
assert entry.end_line == 1 assert entry.end_line == 1
assert entry.end_column == 15 assert entry.end_column == 15
symbol = entry[0]
assert symbol.start_line == 1
assert symbol.start_column == 2
assert symbol.end_line == 1
assert symbol.end_column == 4
inner_expr = entry[1] inner_expr = entry[1]
assert inner_expr.start_line == 1 assert inner_expr.start_line == 1
assert inner_expr.start_column == 6 assert inner_expr.start_column == 6
@ -254,6 +260,20 @@ def test_lex_column_counting():
assert inner_expr.end_column == 14 assert inner_expr.end_column == 14
def test_lex_column_counting_with_literal_newline():
string, symbol = tokenize('"apple\nblueberry" abc')
assert string.start_line == 1
assert string.start_column == 1
assert string.end_line == 2
assert string.end_column == 10
assert symbol.start_line == 2
assert symbol.start_column == 12
assert symbol.end_line == 2
assert symbol.end_column == 14
def test_lex_line_counting_multi(): def test_lex_line_counting_multi():
""" Make sure we can do multi-line tokenization """ """ Make sure we can do multi-line tokenization """
entries = tokenize(""" entries = tokenize("""