better line bits.

hy/compilers/pyast.py

@@ -19,12 +19,11 @@
 # DEALINGS IN THE SOFTWARE.
 
 from hy.compilers import HyCompiler
+from hy.errors import HyError
 
 from hy.models.expression import HyExpression
 from hy.models.symbol import HySymbol
 
-from hy.errors import HyError
-
 import ast
 
 
@@ -63,8 +62,12 @@ class HyASTCompiler(HyCompiler):
                         args=[self.compile(x) for x in expression[1:]],
                         keywords=[],
                         starargs=None,
-                        kwargs=None)
+                        kwargs=None,
+                        lineno=expression.start_line,
+                        col_offset=expression.start_column)
 
     @builds(HySymbol)
     def compile_symbol(self, symbol):
-        return ast.Name(id=str(symbol), ctx=ast.Load())
+        return ast.Name(id=str(symbol), ctx=ast.Load(),
+                        lineno=symbol.start_line,
+                        col_offset=symbol.start_column)

hy/lex/states.py

@@ -109,7 +109,13 @@ class Expression(State):
 
     def commit(self):
         if self.buf != "":
-            self.nodes.append(_resolve_atom(self.buf))
+            ret = _resolve_atom(self.buf)
+            ret.start_line = self._start_line
+            ret.start_column = self._start_column
+            ret.end_line = self.machine.line
+            ret.end_column = (self.machine.column - 1)
+
+            self.nodes.append(ret)
         self.buf = ""
 
     def exit(self):
@@ -139,6 +145,10 @@ class Expression(State):
             self.commit()
             return
 
+        if self.buf == "":
+            self._start_line = self.machine.line
+            self._start_column = self.machine.column
+
         self.buf += char
 
 

tests/lex/test_lex.py

@@ -100,3 +100,18 @@ def test_lex_line_counting_multi():
 
     assert entry.end_line == 3
     assert entry.end_column == 9
+
+
+def test_lex_line_counting_multi_inner():
+    """ Make sure we can do multi-line tokenization (inner) """
+    entry = tokenize("""(foo
+    bar)""")[0]
+    inner = entry[0]
+
+    assert inner.start_line == 1
+    assert inner.start_column == 2
+
+    inner = entry[1]
+
+    assert inner.start_line == 2
+    assert inner.start_column == 5