Merge pull request #1733 from Kodiologist/py2-tokenize

Fix a Python 2 crash

NEWS.rst

@@ -35,6 +35,8 @@ Bug Fixes
   module docstrings caused by automatic importing of Hy builtins.
 * Fixed crash in `mangle` for some pathological inputs
 * Fixed incorrect mangling of some characters at low code points
+* Fixed a crash on certain versions of Python 2 due to changes
+  in the standard module `tokenize`
 
 0.15.0
 ==============================

hy/_compat.py

@@ -47,6 +47,11 @@ def isidentifier(x):
         tokens = list(T.generate_tokens(StringIO(x).readline))
     except (T.TokenError, IndentationError):
         return False
+    # Some versions of Python 2.7 (including one that made it into
+    # Ubuntu 18.10) have a Python 3 backport that adds a NEWLINE
+    # token. Remove it if it's present.
+    # https://bugs.python.org/issue33899
+    tokens = [t for t in tokens if t[0] != T.NEWLINE]
     return len(tokens) == 2 and tokens[0][0] == T.NAME
 
 try: