Merge pull request #1528 from Kodiologist/py37

Python 3.7 support

.travis.yml

@@ -8,12 +8,9 @@ python:
   - "3.7-dev"
   - pypy
   - pypy3
-matrix:
-  allow_failures:
-    - python: "3.7-dev"
 install:
   - pip install -r requirements-travis.txt
-  - pip install -e .
+  - pip install --process-dependency-links -e .
 script: pytest
 cache: pip
 after_success: make coveralls

NEWS.rst

@@ -18,6 +18,7 @@ Other Breaking Changes
 
 New Features
 ------------------------------
+* Python 3.7 is now supported
 * Added `mangle` and `unmangle` as core functions
 * `defclass` in Python 3 now supports specifying metaclasses and other
   keyword arguments

hy/compiler.py

@@ -12,7 +12,7 @@ from hy.lex.parser import mangle
 
 import hy.macros
 from hy._compat import (
-    str_type, string_types, bytes_type, long_type, PY3, PY35,
+    str_type, string_types, bytes_type, long_type, PY3, PY35, PY37,
     raise_empty)
 from hy.macros import require, macroexpand, tag_macroexpand
 import hy.importer
@@ -1949,10 +1949,11 @@ class HyASTCompiler(object):
                 ) + expression
                 expression = expression.replace(arg[0])
 
-        # Docstrings must come at the start, so ensure that happens even if we
-        # generate anonymous variables.
-        if docstring is not None:
+        # Before Python 3.7, docstrings must come at the start, so ensure that
+        # happens even if we generate anonymous variables.
+        if docstring is not None and not PY37:
             expression.insert(0, docstring)
+            docstring = None
 
         if PY3:
             # Python 3.4+ requires that args are an ast.arg object, rather
@@ -1989,7 +1990,7 @@ class HyASTCompiler(object):
             defaults=defaults)
 
         body = self._compile_branch(expression)
-        if not force_functiondef and not body.stmts:
+        if not force_functiondef and not body.stmts and docstring is None:
             ret += asty.Lambda(expression, args=args, body=body.force_expr)
             return ret
 
@@ -2012,7 +2013,9 @@ class HyASTCompiler(object):
                     name=name,
                     args=args,
                     body=body.stmts,
-                    decorator_list=[])
+                    decorator_list=[],
+                    docstring=(None if docstring is None else
+                        str_type(docstring)))
 
         ast_name = asty.Name(expression, id=name, ctx=ast.Load())
 
@@ -2225,9 +2228,16 @@ def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
     if not get_expr:
         result += result.expr_as_stmt()
 
+    module_docstring = None
+    if (PY37 and result.stmts and
+            isinstance(result.stmts[0], ast.Expr) and
+            isinstance(result.stmts[0].value, ast.Str)):
+        module_docstring = result.stmts.pop(0).value.s
+
     body = compiler.imports_as_stmts(tree) + result.stmts
 
-    ret = root(body=body)
+    ret = root(body=body, docstring=(
+        None if module_docstring is None else module_docstring))
 
     if get_expr:
         expr = ast.Expression(body=expr)

hy/core/language.hy

@@ -8,7 +8,6 @@
 
 (import itertools)
 (import functools)
-(import collections)
 (import [fractions [Fraction :as fraction]])
 (import operator)  ; shadow not available yet
 (import sys)
@@ -16,6 +15,9 @@
   (import [StringIO [StringIO]])
   (import [io [StringIO]]))
 (import [hy._compat [long-type]]) ; long for python2, int for python3
+(if-python2
+  (import [collections :as cabc])
+  (import [collections.abc :as cabc]))
 (import [hy.models [HyCons HySymbol HyKeyword]])
 (import [hy.lex [LexException PrematureEndOfInput tokenize]])
 (import [hy.lex.parser [mangle unmangle]])
@@ -278,7 +280,7 @@ Return series of accumulated sums (or other binary function results)."
 
 (defn iterable? [x]
   "Check if `x` is an iterable."
-  (isinstance x collections.Iterable))
+  (isinstance x cabc.Iterable))
 
 (defn iterate [f x]
   "Returns an iterator repeatedly applying `f` to seed `x`.. x, f(x), f(f(x))..."
@@ -289,7 +291,7 @@ Return series of accumulated sums (or other binary function results)."
 
 (defn iterator? [x]
   "Check if `x` is an iterator."
-  (isinstance x collections.Iterator))
+  (isinstance x cabc.Iterator))
 
 (defn juxt [f &rest fs]
   "Return a function applying each `fs` to args, collecting results in a list."

hy/importer.py

@@ -77,6 +77,9 @@ def import_file_to_module(module_name, fpath, loader=None):
             # The first 4 bytes are the magic number for the version of Python
             # that compiled this bytecode.
             bytecode_magic = bc_f.read(4)
+            # Python 3.7 introduced a new flags entry in the header structure.
+            if PY37:
+                bc_f.read(4)
             # The next 4 bytes, interpreted as a little-endian 32-bit integer,
             # are the mtime of the corresponding source file.
             bytecode_mtime, = struct.unpack('<i', bc_f.read(4))

setup.py

@@ -30,7 +30,7 @@ class Install(install):
                         "." + filename[:-len(".hy")])
         install.run(self)
 
-install_requires = ['rply>=0.7.5', 'astor>=0.6', 'clint>=0.4']
+install_requires = ['rply>=0.7.5', 'astor', 'clint>=0.4']
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 
@@ -40,6 +40,9 @@ setup(
     name=PKG,
     version=__version__,
     install_requires=install_requires,
+    dependency_links=[
+        'git+https://github.com/berkerpeksag/astor.git#egg=astor-0.7.0'
+    ],
     cmdclass=dict(install=Install),
     entry_points={
         'console_scripts': [
@@ -81,6 +84,7 @@ setup(
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Topic :: Software Development :: Code Generators",
         "Topic :: Software Development :: Compilers",
         "Topic :: Software Development :: Libraries",

tests/compilers/test_ast.py

@@ -53,7 +53,7 @@ def cant_compile(expr):
 
 
 def s(x):
-    return can_compile(x).body[0].value.s
+    return can_compile('"module docstring" ' + x).body[-1].value.s
 
 
 def test_ast_bad_type():
@@ -476,13 +476,12 @@ def test_ast_unicode_strings():
 
     def _compile_string(s):
         hy_s = HyString(s)
-        hy_s.start_line = hy_s.end_line = 0
-        hy_s.start_column = hy_s.end_column = 0
 
-        code = hy_compile(hy_s, "__main__")
+        code = hy_compile([hy_s], "__main__")
+        # We put hy_s in a list so it isn't interpreted as a docstring.
 
-        # code == ast.Module(body=[ast.Expr(value=ast.Str(s=xxx))])
-        return code.body[0].value.s
+        # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))])
+        return code.body[0].value.elts[0].s
 
     assert _compile_string("test") == "test"
     assert _compile_string("\u03b1\u03b2") == "\u03b1\u03b2"

tests/native_tests/contrib/hy_repr.hy

@@ -3,7 +3,7 @@
 ;; license. See the LICENSE.
 
 (import
-  [hy._compat [PY3 PY36]]
+  [hy._compat [PY3 PY36 PY37]]
   [math [isnan]]
   [hy.contrib.hy-repr [hy-repr hy-repr-register]])
 
@@ -163,8 +163,8 @@
   (setv mo (re.search "b+" "aaaabbbccc"))
   (assert (= (hy-repr mo)
     (.format
-      #[[<{}.SRE_Match object; :span {} :match "bbb">]]
-      (. (type mo) __module__)
+      #[[<{} object; :span {} :match "bbb">]]
+      (if PY37 "re.Match" (+ (. (type mo) __module__) ".SRE_Match"))
       (if PY3 "(, 4 7)" "(, (int 4) (int 7))")))))
 
 (defn test-hy-repr-custom []

tests/native_tests/language.hy

@@ -11,7 +11,7 @@
         pytest)
 (import sys)
 
-(import [hy._compat [PY3 PY35]])
+(import [hy._compat [PY3 PY35 PY37]])
 
 (defn test-sys-argv []
   "NATIVE: test sys.argv"
@@ -1606,18 +1606,21 @@
 
 (defn test-disassemble []
   "NATIVE: Test the disassemble function"
-  (if PY35
-    (assert (= (disassemble '(do (leaky) (leaky) (macros)))
-               "Module(
+  (assert (= (disassemble '(do (leaky) (leaky) (macros))) (cond
+    [PY37 "Module(
     body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
-        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))])"))
-    (assert (= (disassemble '(do (leaky) (leaky) (macros)))
-               "Module(
+        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))],
+    docstring=None)"]
+    [PY35 "Module(
+    body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
+        Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
+        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))])"]
+    [True "Module(
     body=[
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),
-        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[], starargs=None, kwargs=None))])")))
+        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[], starargs=None, kwargs=None))])"])))
   (assert (= (disassemble '(do (leaky) (leaky) (macros)) True)
              "leaky()
 leaky()
@@ -1785,6 +1788,11 @@ macros()
   (assert (none? (. f4 __doc__)))
   (assert (= (f4 [1 2]) "not a docstring")))
 
+(defn test-module-docstring []
+  (import [tests.resources.module-docstring-example :as m])
+  (assert (= m.__doc__ "This is the module docstring."))
+  (assert (= m.foo 5)))
+
 (defn test-relative-import []
   "Make sure relative imports work properly"
   (import [..resources [tlib]])

tests/resources/module_docstring_example.hy

@@ -0,0 +1,3 @@
+"This is the module docstring."
+
+(setv foo 5)