diff --git a/NEWS.rst b/NEWS.rst index 518ca7a..6f3d53c 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -35,6 +35,8 @@ Bug Fixes module docstrings caused by automatic importing of Hy builtins. * Fixed crash in `mangle` for some pathological inputs * Fixed incorrect mangling of some characters at low code points +* Fixed a crash on certain versions of Python 2 due to changes + in the standard module `tokenize` 0.15.0 ============================== diff --git a/hy/_compat.py b/hy/_compat.py index 379ce43..ddd6c48 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -47,6 +47,11 @@ def isidentifier(x): tokens = list(T.generate_tokens(StringIO(x).readline)) except (T.TokenError, IndentationError): return False + # Some versions of Python 2.7 (including one that made it into + # Ubuntu 18.10) have a Python 3 backport that adds a NEWLINE + # token. Remove it if it's present. + # https://bugs.python.org/issue33899 + tokens = [t for t in tokens if t[0] != T.NEWLINE] return len(tokens) == 2 and tokens[0][0] == T.NAME try: