From f1e693c96b4dbd03f5b702c55f08a04685abbf2a Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 20 Jan 2019 11:59:49 -0500
Subject: [PATCH] Fix a Python 2 crash

---
 NEWS.rst      | 2 ++
 hy/_compat.py | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 518ca7a..6f3d53c 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -35,6 +35,8 @@ Bug Fixes
   module docstrings caused by automatic importing of Hy builtins.
 * Fixed crash in `mangle` for some pathological inputs
 * Fixed incorrect mangling of some characters at low code points
+* Fixed a crash on certain versions of Python 2 due to changes
+  in the standard module `tokenize`
 
 0.15.0
 ==============================
diff --git a/hy/_compat.py b/hy/_compat.py
index 379ce43..ddd6c48 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -47,6 +47,11 @@ def isidentifier(x):
         tokens = list(T.generate_tokens(StringIO(x).readline))
     except (T.TokenError, IndentationError):
         return False
+    # Some versions of Python 2.7 (including one that made it into
+    # Ubuntu 18.10) have a Python 3 backport that adds a NEWLINE
+    # token. Remove it if it's present.
+    # https://bugs.python.org/issue33899
+    tokens = [t for t in tokens if t[0] != T.NEWLINE]
     return len(tokens) == 2 and tokens[0][0] == T.NAME
 
 try: