Mangle names that coincide with Python keywords

hy/_compat.py

@@ -18,7 +18,7 @@ except ImportError:
                        (x >> 8) & 0xff,
                        (x >> 16) & 0xff,
                        (x >> 24) & 0xff]))
-import sys
+import sys, keyword
 
 PY3 = sys.version_info[0] >= 3
 PY35 = sys.version_info >= (3, 5)
@@ -37,15 +37,22 @@ else:
         raise t(*args)
 
 def isidentifier(x):
+    if x in ('True', 'False', 'None', 'print'):
+        # `print` is special-cased here because Python 2's
+        # keyword.iskeyword will count it as a keyword, but we
+        # use the __future__ feature print_function, which makes
+        # it a non-keyword.
+        return True
+    if keyword.iskeyword(x):
+        return False
     if PY3:
         return x.isidentifier()
-    else:
-        if x.rstrip() != x:
-            return False
-        import tokenize as T
-        from StringIO import StringIO
-        try:
-            tokens = list(T.generate_tokens(StringIO(x).readline))
-        except T.TokenError:
-            return False
-        return len(tokens) == 2 and tokens[0][0] == T.NAME
+    if x.rstrip() != x:
+        return False
+    import tokenize as T
+    from StringIO import StringIO
+    try:
+        tokens = list(T.generate_tokens(StringIO(x).readline))
+    except T.TokenError:
+        return False
+    return len(tokens) == 2 and tokens[0][0] == T.NAME

hy/compiler.py

@@ -61,20 +61,6 @@ def load_stdlib():
                 _stdlib[e] = module
 
 
-# True, False and None included here since they
-# are assignable in Python 2.* but become
-# keywords in Python 3.*
-def _is_hy_builtin(name, module_name):
-    extras = ['True', 'False', 'None']
-    if name in extras or keyword.iskeyword(name):
-        return True
-    # for non-Hy modules, check for pre-existing name in
-    # _compile_table
-    if not module_name.startswith("hy."):
-        return name in _compile_table
-    return False
-
-
 _compile_table = {}
 _decoratables = (ast.FunctionDef, ast.ClassDef)
 if PY35:
@@ -386,7 +372,6 @@ def ends_with_else(expr):
 class HyASTCompiler(object):
 
     def __init__(self, module_name):
-        self.allow_builtins = module_name.startswith("hy.core")
         self.anon_var_count = 0
         self.imports = defaultdict(set)
         self.module_name = module_name
@@ -1803,10 +1788,11 @@ class HyASTCompiler(object):
     def _compile_assign(self, name, result):
 
         str_name = "%s" % name
-        if (_is_hy_builtin(str_name, self.module_name) and
-                not self.allow_builtins):
+        if str_name in (["None"] + (["True", "False"] if PY3 else [])):
+            # Python 2 allows assigning to True and False, although
+            # this is rarely wise.
             raise HyTypeError(name,
-                              "Can't assign to a builtin: `%s'" % str_name)
+                              "Can't assign to `%s'" % str_name)
 
         result = self.compile(result)
         ld_name = self.compile(name)
@@ -2082,8 +2068,6 @@ class HyASTCompiler(object):
             body += self._compile_assign(symb, docstring)
             body += body.expr_as_stmt()
 
-        allow_builtins = self.allow_builtins
-        self.allow_builtins = True
         if expressions and isinstance(expressions[0], HyList) \
            and not isinstance(expressions[0], HyExpression):
             expr = expressions.pop(0)
@@ -2095,8 +2079,6 @@ class HyASTCompiler(object):
         for expression in expressions:
             body += self.compile(rewire_init(macroexpand(expression, self)))
 
-        self.allow_builtins = allow_builtins
-
         if not body.stmts:
             body += asty.Pass(expressions)
 

tests/compilers/test_ast.py

@@ -596,13 +596,11 @@ def test_invalid_list_comprehension():
 
 def test_bad_setv():
     """Ensure setv handles error cases"""
-    cant_compile("(setv if* 1)")
     cant_compile("(setv (a b) [1 2])")
 
 
 def test_defn():
     """Ensure that defn works correctly in various corner cases"""
-    cant_compile("(defn if* [] 1)")
     cant_compile("(defn \"hy\" [] 1)")
     cant_compile("(defn :hy [] 1)")
     can_compile("(defn &hy [] 1)")
@@ -611,7 +609,6 @@ def test_defn():
 def test_setv_builtins():
     """Ensure that assigning to a builtin fails, unless in a class"""
     cant_compile("(setv None 42)")
-    cant_compile("(defn get [&rest args] 42)")
     can_compile("(defclass A [] (defn get [self] 42))")
     can_compile("""
     (defclass A []

tests/native_tests/language.hy

@@ -65,19 +65,19 @@
 
 
 (defn test-setv-builtin []
-  "NATIVE: test that setv doesn't work on builtins"
-  (try (eval '(setv False 1))
-       (except [e [TypeError]] (assert (in "Can't assign to a builtin" (str e)))))
-  (try (eval '(setv True 0))
-       (except [e [TypeError]] (assert (in "Can't assign to a builtin" (str e)))))
+  "NATIVE: test that setv doesn't work on names Python can't assign to
+  and that we can't mangle"
   (try (eval '(setv None 1))
-       (except [e [TypeError]] (assert (in "Can't assign to a builtin" (str e)))))
-  (try (eval '(defn defclass [] (print "hello")))
-       (except [e [TypeError]] (assert (in "Can't assign to a builtin" (str e)))))
-  (try (eval '(defn get [] (print "hello")))
-       (except [e [TypeError]] (assert (in "Can't assign to a builtin" (str e)))))
-  (try (eval '(defn fn [] (print "hello")))
-       (except [e [TypeError]] (assert (in "Can't assign to a builtin" (str e))))))
+       (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+  (try (eval '(defn None [] (print "hello")))
+       (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+  (when PY3
+    (try (eval '(setv False 1))
+         (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+    (try (eval '(setv True 0))
+         (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+    (try (eval '(defn True [] (print "hello")))
+         (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))))
 
 
 (defn test-setv-pairs []
@@ -223,14 +223,14 @@
 
   ; don't be fooled by constructs that look like else
   (setv s "")
-  (setv (get (globals) "else") True)
+  (setv else True)
   (for [x "abcde"]
     (+= s x)
     [else (+= s "_")])
   (assert (= s "a_b_c_d_e_"))
 
   (setv s "")
-  (setv (get (globals) "else") True)
+  (setv else True)
   (with [(pytest.raises TypeError)]
     (for [x "abcde"]
       (+= s x)
@@ -329,7 +329,7 @@
   ; don't be fooled by constructs that look like else clauses
   (setv x 2)
   (setv a [])
-  (setv (get (globals) "else") True)
+  (setv else True)
   (while x
     (.append a x)
     (-= x 1)

tests/native_tests/mangling.hy

@@ -115,6 +115,20 @@
   (assert (= x "aabb")))
 
 
+(defn test-python-keyword []
+  (setv if 3)
+  (assert (= if 3))
+  (assert (= hyx_if 3)))
+
+
+(defn test-operator []
+  (setv + 3)
+  (assert (= + 3))
+  (if PY3
+    (assert (= hyx_Δplus_signΔ 3))
+    (assert (= hyx_Xplus_signX 3))))
+
+
 (defn test-late-mangling []
   ; Mangling should only happen during compilation.
   (assert (!= 'foo? 'is_foo))