Allow commas and underscores in numeric literals

You can use them as thousands separators.

This change differs from PEP 515 in that not only does it allow commas in addition to underscores, but it's much more liberal about placement. Any number of underscores or commas can be placed anywhere, even at the start.

NEWS

@@ -3,6 +3,7 @@ Changes from 0.12.1
    [ Language Changes ]
    * `let` has been removed. Python's scoping rules do not make a proper
      implementation of it possible. Use `setv` instead.
+   * Commas and underscores are allowed in numeric literals
    * xor: If exactly one argument is true, return it
 
    [ Bug Fixes ]

docs/language/api.rst

@@ -18,7 +18,7 @@ These rules help ensure that Hy code is idiomatic and interfaceable in both
 languages.
 
 
-  * Symbols in earmufs will be translated to the upper-cased version of that
+  * Symbols in earmuffs will be translated to the upper-cased version of that
     string. For example, ``foo`` will become ``FOO``.
 
   * UTF-8 entities will be encoded using
@@ -34,10 +34,8 @@ languages.
 Notes on Syntax
 ===============
 
-integers
+numeric literals
---------
+----------------
-
-.. versionadded:: 0.11.1
 
 In addition to regular numbers, standard notation from Python 3 for non-base 10
 integers is used. ``0x`` for Hex, ``0o`` for Octal, ``0b`` for Binary.
@@ -46,6 +44,13 @@ integers is used. ``0x`` for Hex, ``0o`` for Octal, ``0b`` for Binary.
     
     (print 0x80 0b11101 0o102 30)
 
+Underscores and commas can appear anywhere in a numeric literal. They have no
+effect on the value of the literal, but they're useful for visually separating
+digits.
+
+.. code-block:: clj
+
+    (print 10,000,000,000 10_000_000_000)
 
 Built-Ins
 =========

hy/models/complex.py

@@ -19,6 +19,7 @@
 # DEALINGS IN THE SOFTWARE.
 
 from hy.models import HyObject, _wrappers
+from hy._compat import string_types
 
 
 class HyComplex(HyObject, complex):
@@ -28,6 +29,8 @@ class HyComplex(HyObject, complex):
     """
 
     def __new__(cls, number, *args, **kwargs):
+        if isinstance(number, string_types):
+            number = number.replace("_", "").replace(",", "")
         number = complex(number)
         return super(HyComplex, cls).__new__(cls, number)
 

hy/models/float.py

@@ -19,6 +19,7 @@
 # DEALINGS IN THE SOFTWARE.
 
 from hy.models import HyObject, _wrappers
+from hy._compat import string_types
 
 
 class HyFloat(HyObject, float):
@@ -28,6 +29,8 @@ class HyFloat(HyObject, float):
     """
 
     def __new__(cls, number, *args, **kwargs):
+        if isinstance(number, string_types):
+            number = number.replace("_", "").replace(",", "")
         number = float(number)
         return super(HyFloat, cls).__new__(cls, number)
 

hy/models/integer.py

@@ -19,7 +19,7 @@
 # DEALINGS IN THE SOFTWARE.
 
 from hy.models import HyObject, _wrappers
-from hy._compat import long_type, str_type
+from hy._compat import long_type, string_types
 
 import sys
 
@@ -32,7 +32,8 @@ class HyInteger(HyObject, long_type):
     """
 
     def __new__(cls, number, *args, **kwargs):
-        if isinstance(number, str_type):
+        if isinstance(number, string_types):
+            number = number.replace("_", "").replace(",", "")
             bases = {"0x": 16, "0o": 8, "0b": 2}
             for leader, base in bases.items():
                 if number.startswith(leader):

tests/lex/test_lex.py

@@ -143,6 +143,32 @@ def test_lex_expression_complex():
     assert objs == [HyExpression([HySymbol("foo"), HySymbol("j")])]
 
 
+def test_lex_digit_separators():
+
+    assert tokenize("1_000_000") == [HyInteger(1000000)]
+    assert tokenize("1,000,000") == [HyInteger(1000000)]
+    assert tokenize("1,000_000") == [HyInteger(1000000)]
+    assert tokenize("1_000,000") == [HyInteger(1000000)]
+
+    assert tokenize("0x_af") == [HyInteger(0xaf)]
+    assert tokenize("0x,af") == [HyInteger(0xaf)]
+    assert tokenize("0b_010") == [HyInteger(0b010)]
+    assert tokenize("0b,010") == [HyInteger(0b010)]
+    assert tokenize("0o_373") == [HyInteger(0o373)]
+    assert tokenize("0o,373") == [HyInteger(0o373)]
+
+    assert tokenize('1_2.3,4') == [HyFloat(12.34)]
+    assert tokenize('1_2e3,4') == [HyFloat(12e34)]
+    assert (tokenize("1,2/3_4") ==
+            [HyExpression([HySymbol("fraction"),
+             HyInteger(12), HyInteger(34)])])
+    assert tokenize("1,0_00j") == [HyComplex(1000j)]
+
+    assert tokenize(",,,,___,__1__,,__,,2__,,,__") == [HyInteger(12)]
+    assert (tokenize(",,,,___,__1__,,__,,2__,q,__") ==
+            [HySymbol(",,,,___,__1__,,__,,2__,q,__")])
+
+
 def test_lex_line_counting():
     """ Make sure we can count lines / columns """
     entry = tokenize("(foo (one two))")[0]