From a44e53f4de38b02726b8348505a112999d3b8411 Mon Sep 17 00:00:00 2001 From: Nicolas Dandrimont Date: Thu, 2 Jan 2014 03:09:18 +0100 Subject: [PATCH 1/2] Comments end when the input ends or a newline occurs This fixes #382, which occured because the REPL doesn't use trailing newlines. --- hy/lex/lexer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hy/lex/lexer.py b/hy/lex/lexer.py index d4a5638..8032fc0 100644 --- a/hy/lex/lexer.py +++ b/hy/lex/lexer.py @@ -60,7 +60,7 @@ lg.add('STRING', r'''(?x) lg.add('IDENTIFIER', r'[^()\[\]{}\'"\s;]+') -lg.ignore(r';.*[\r\n]+') +lg.ignore(r';.*(?=\r|\n|$)') lg.ignore(r'\s+') From c1d5948d73280e61e48cf42f9aed944e87392aa3 Mon Sep 17 00:00:00 2001 From: Nicolas Dandrimont Date: Thu, 2 Jan 2014 03:13:49 +0100 Subject: [PATCH 2/2] Add regression test for #382 --- tests/lex/test_lex.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/lex/test_lex.py b/tests/lex/test_lex.py index bda1e3f..af9f286 100644 --- a/tests/lex/test_lex.py +++ b/tests/lex/test_lex.py @@ -260,3 +260,9 @@ def test_reader_macro(): assert entry[0][0] == HySymbol("dispatch_reader_macro") assert entry[0][1] == HyExpression([HySymbol("quote"), HyString("^")]) assert len(entry[0]) == 3 + + +def test_lex_comment_382(): + """Ensure that we can tokenize sources with a comment at the end""" + entry = tokenize("foo ;bar\n;baz") + assert entry == [HySymbol("foo")]