From 8296a36e125bb1e7ffba9301818145a191f64395 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 28 May 2018 16:15:47 -0700
Subject: [PATCH 001/223] Add tests and docs for model patterns

---
 NEWS.rst                             |   1 +
 docs/language/index.rst              |   1 +
 docs/language/model_patterns.rst     | 113 +++++++++++++++++++++++++++
 tests/native_tests/model_patterns.hy |  70 +++++++++++++++++
 4 files changed, 185 insertions(+)
 create mode 100644 docs/language/model_patterns.rst
 create mode 100644 tests/native_tests/model_patterns.hy

diff --git a/NEWS.rst b/NEWS.rst
index 5b2d54c..35155f2 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -47,6 +47,7 @@ New Features
   keyword arguments
 * Added a command-line option `-E` per CPython
 * `while` and `for` are allowed to have empty bodies
+* Added a new module ``hy.model_patterns``
 
 Bug Fixes
 ------------------------------
diff --git a/docs/language/index.rst b/docs/language/index.rst
index 1a73bd2..abfe290 100644
--- a/docs/language/index.rst
+++ b/docs/language/index.rst
@@ -12,4 +12,5 @@ Contents:
    syntax
    api
    core
+   model_patterns
    internals
diff --git a/docs/language/model_patterns.rst b/docs/language/model_patterns.rst
new file mode 100644
index 0000000..a099850
--- /dev/null
+++ b/docs/language/model_patterns.rst
@@ -0,0 +1,113 @@
+==============
+Model Patterns
+==============
+
+The module ``hy.model-patterns`` provides a library of parser combinators for
+parsing complex trees of Hy models. Model patterns exist mostly to help
+implement the compiler, but they can also be useful for writing macros.
+
+A motivating example
+--------------------
+
+The kind of problem that model patterns are suited for is the following.
+Suppose you want to validate and extract the components of a form like:
+
+.. code-block:: clj
+
+    (setv form '(try
+      (foo1)
+      (foo2)
+      (except [EType1]
+        (foo3))
+      (except [e EType2]
+        (foo4)
+        (foo5))
+      (except []
+        (foo6))
+      (finally
+        (foo7)
+        (foo8))))
+
+You could do this with loops and indexing, but it would take a lot of code and
+be error-prone. Model patterns concisely express the general form of an
+expression to be matched, like what a regular expression does for text. Here's
+a pattern for a ``try`` form of the above kind:
+
+.. code-block:: clj
+
+    (import [funcparserlib.parser [maybe many]])
+    (import [hy.model-patterns [*]])
+    (setv parser (whole [
+      (sym "try")
+      (many (notpexpr "except" "else" "finally"))
+      (many (pexpr
+        (sym "except")
+        (| (brackets) (brackets FORM) (brackets SYM FORM))
+        (many FORM)))
+      (maybe (dolike "else"))
+      (maybe (dolike "finally"))]))
+
+You can run the parser with ``(.parse parser form)``. The result is:
+
+.. code-block:: clj
+
+    (,
+      ['(foo1) '(foo2)]
+      [
+        '([EType1] [(foo3)])
+        '([e EType2] [(foo4) (foo5)])
+        '([] [(foo6)])]
+      None
+      '((foo7) (foo8)))
+
+which is conveniently utilized with an assignment such as ``(setv [body
+except-clauses else-part finally-part] result)``. Notice that ``else-part``
+will be set to ``None`` because there is no ``else`` clause in the original
+form.
+
+Usage
+-----
+
+Model patterns are implemented as funcparserlib_ parser combinators. We won't
+reproduce funcparserlib's own documentation, but here are some important
+built-in parsers:
+
+- ``(+ ...)`` matches its arguments in sequence.
+- ``(| ...)`` matches any one of its arguments.
+- ``(>> parser function)`` matches ``parser``, then feeds the result through
+  ``function`` to change the value that's produced on a successful parse.
+- ``(skip parser)`` matches ``parser``, but doesn't add it to the produced
+  value.
+- ``(maybe parser)`` matches ``parser`` if possible. Otherwise, it produces
+  the value ``None``.
+- ``(some function)`` takes a predicate ``function`` and matches a form if it
+  satisfies the predicate.
+
+The best reference for Hy's parsers is the docstrings (use ``(help
+hy.model-patterns)``), but again, here are some of the more important ones:
+
+- ``FORM`` matches anything.
+- ``SYM`` matches any symbol.
+- ``(sym "foo")`` or ``(sym ":foo")`` matches and discards (per ``skip``) the
+  named symbol or keyword.
+- ``(brackets ...)`` matches the arguments in square brackets.
+- ``(pexpr ...)`` matches the arguments in parentheses.
+
+Here's how you could write a simple macro using model patterns:
+
+.. code-block:: clj
+
+    (defmacro pairs [&rest args]
+      (import [funcparserlib.parser [many]])
+      (import [hy.model-patterns [whole SYM FORM]])
+      (setv [args] (->> args (.parse (whole [
+        (many (+ SYM FORM))]))))
+      `[~@(->> args (map (fn [x]
+        (, (name (get x 0)) (get x 1)))))])
+
+    (print (pairs  a 1  b 2  c 3))
+    ; => [["a" 1] ["b" 2] ["c" 3]]
+
+A failed parse will raise ``funcparserlib.parser.NoParseError``.
+
+.. _funcparserlib: https://github.com/vlasovskikh/funcparserlib
diff --git a/tests/native_tests/model_patterns.hy b/tests/native_tests/model_patterns.hy
new file mode 100644
index 0000000..0e6c316
--- /dev/null
+++ b/tests/native_tests/model_patterns.hy
@@ -0,0 +1,70 @@
+;; Copyright 2018 the authors.
+;; This file is part of Hy, which is free software licensed under the Expat
+;; license. See the LICENSE.
+
+(defmacro do-until [&rest args]
+  (import
+    [hy.model-patterns [whole FORM notpexpr dolike]]
+    [funcparserlib.parser [many]])
+  (setv [body condition] (->> args (.parse (whole
+    [(many (notpexpr "until")) (dolike "until")]))))
+  (setv g (gensym))
+  `(do
+    (setv ~g True)
+    (while (or ~g (not (do ~@condition)))
+      ~@body
+      (setv ~g False))))
+
+(defn test-do-until []
+  (setv n 0  s "")
+  (do-until
+    (+= s "x")
+    (until (+= n 1) (>= n 3)))
+  (assert (= s "xxx"))
+  (do-until
+    (+= s "x")
+    (until (+= n 1) (>= n 3)))
+  (assert (= s "xxxx")))
+
+(defmacro loop [&rest args]
+  (import
+    [hy.model-patterns [whole FORM sym SYM]]
+    [funcparserlib.parser [many]])
+  (setv [loopers body] (->> args (.parse (whole [
+    (many (|
+      (>> (+ (sym "while") FORM) (fn [x] [x]))
+      (+ (sym "for") SYM (sym "in") FORM)
+      (+ (sym "for") SYM (sym "from") FORM (sym "to") FORM)))
+    (sym "do")
+    (many FORM)]))))
+  (defn f [loopers]
+    (setv [head tail] [(first loopers) (cut loopers 1)])
+    (print head)
+    (cond
+      [(none? head)
+        `(do ~@body)]
+      [(= (len head) 1)
+        `(while ~@head ~(f tail))]
+      [(= (len head) 2)
+        `(for [~@head] ~(f tail))]
+      [True ; (= (len head) 3)
+        (setv [sym from to] head)
+        `(for [~sym (range ~from (inc ~to))] ~(f tail))]))
+  (f loopers))
+
+(defn test-loop []
+
+  (setv l [])
+  (loop
+     for x in "abc"
+     do (.append l x))
+  (assert (= l ["a" "b" "c"]))
+
+  (setv l []  k 2)
+  (loop
+     while (> k 0)
+     for n from 1 to 3
+     for p in [k n (* 10 n)]
+     do (.append l p) (-= k 1))
+  (print l)
+  (assert (= l [2 1 10  -1 2 20  -4 3 30])))

From bc2a5a2747b2caf64d75012443c39e939b17fc0c Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 5 Jun 2018 17:32:48 -0700
Subject: [PATCH 002/223] Don't test on Travis's outdated Python 3.7

See https://github.com/travis-ci/travis-ci/issues/9069 .
---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4538c7c..81ea5b8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,6 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-  - "3.7-dev"
   - pypy
   - pypy3
 install:

From 16ec46a4739fe243113568324ff2036aa3469ac9 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 5 Jun 2018 17:35:48 -0700
Subject: [PATCH 003/223] Update docstring handling for Python 3.7

See https://github.com/python/cpython/pull/7121 .
---
 hy/compiler.py                 | 38 ++++++----------------------------
 tests/native_tests/language.hy |  5 -----
 2 files changed, 6 insertions(+), 37 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 5bdc025..387b23d 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1377,9 +1377,8 @@ class HyASTCompiler(object):
             maybe(sym("&rest") + NASYM),
             maybe(sym("&kwonly") + many(NASYM | brackets(SYM, FORM))),
             maybe(sym("&kwargs") + NASYM)),
-        maybe(STR),
         many(FORM)])
-    def compile_function_def(self, expr, root, params, docstring, body):
+    def compile_function_def(self, expr, root, params, body):
 
         force_functiondef = root in ("fn*", "fn/a")
         node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef
@@ -1411,22 +1410,9 @@ class HyASTCompiler(object):
             kwonlyargs=kwonly, kw_defaults=kw_defaults,
             kwarg=kwargs)
 
-        if docstring is not None:
-            if not body:
-                # Reinterpret the docstring as the return value of the
-                # function. Thus, (fn [] "hello") returns "hello" and has no
-                # docstring, instead of returning None and having a docstring
-                # "hello".
-                body = [docstring]
-                docstring = None
-            elif not PY37:
-                # The docstring needs to be represented in the AST as a body
-                # statement.
-                body = [docstring] + body
-                docstring = None
         body = self._compile_branch(body)
 
-        if not force_functiondef and not body.stmts and docstring is None:
+        if not force_functiondef and not body.stmts:
             return ret + asty.Lambda(expr, args=args, body=body.force_expr)
 
         if body.expr:
@@ -1444,9 +1430,7 @@ class HyASTCompiler(object):
                     name=name,
                     args=args,
                     body=body.stmts or [asty.Pass(expr)],
-                    decorator_list=[],
-                    docstring=(None if docstring is None else
-                        str_type(docstring)))
+                    decorator_list=[])
 
         ast_name = asty.Name(expr, id=name, ctx=ast.Load())
         ret += Result(expr=ast_name, temp_variables=[ast_name, ret.stmts[-1]])
@@ -1489,9 +1473,7 @@ class HyASTCompiler(object):
         bodyr = Result()
 
         if docstring is not None:
-            if not PY37:
-                bodyr += self.compile(docstring).expr_as_stmt()
-                docstring = None
+            bodyr += self.compile(docstring).expr_as_stmt()
 
         if attrs is not None:
             bodyr += self.compile(self._rewire_init(HyExpression(
@@ -1510,8 +1492,7 @@ class HyASTCompiler(object):
             starargs=None,
             kwargs=None,
             bases=bases_expr,
-            body=bodyr.stmts or [asty.Pass(expr)],
-            docstring=(None if docstring is None else str_type(docstring)))
+            body=bodyr.stmts or [asty.Pass(expr)])
 
     def _rewire_init(self, expr):
         "Given a (setv …) form, append None to definitions of __init__."
@@ -1736,16 +1717,9 @@ def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
     if not get_expr:
         result += result.expr_as_stmt()
 
-    module_docstring = None
-    if (PY37 and result.stmts and
-            isinstance(result.stmts[0], ast.Expr) and
-            isinstance(result.stmts[0].value, ast.Str)):
-        module_docstring = result.stmts.pop(0).value.s
-
     body = compiler.imports_as_stmts(tree) + result.stmts
 
-    ret = root(body=body, docstring=(
-        None if module_docstring is None else module_docstring))
+    ret = root(body=body)
 
     if get_expr:
         expr = ast.Expression(body=expr)
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 98e9978..7e44304 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1625,11 +1625,6 @@
 (defn test-disassemble []
   "NATIVE: Test the disassemble function"
   (assert (= (disassemble '(do (leaky) (leaky) (macros))) (cond
-    [PY37 "Module(
-    body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
-        Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
-        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))],
-    docstring=None)"]
     [PY35 "Module(
     body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),

From ea899471af5caccdf27089c0a2e1c4dab0bfe1e9 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 29 May 2018 11:47:46 -0700
Subject: [PATCH 004/223] Remove an unused compiler function

---
 hy/compiler.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 387b23d..3d584e6 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -328,14 +328,6 @@ def is_unpack(kind, x):
             and x[0] == "unpack-" + kind)
 
 
-def ends_with_else(expr):
-    return (expr and
-            isinstance(expr[-1], HyExpression) and
-            expr[-1] and
-            isinstance(expr[-1][0], HySymbol) and
-            expr[-1][0] == HySymbol("else"))
-
-
 class HyASTCompiler(object):
 
     def __init__(self, module_name):

From c3d4c7aa82aece3e05ff7fc47de5c63df445b672 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 29 May 2018 13:26:02 -0700
Subject: [PATCH 005/223] Clean up `else` compilation in `while`

---
 hy/compiler.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 3d584e6..fe47c9b 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1342,9 +1342,8 @@ class HyASTCompiler(object):
 
         orel = Result()
         if else_expr is not None:
-            for else_body in else_expr:
-                orel += self.compile(else_body)
-                orel += orel.expr_as_stmt()
+            orel = self._compile_branch(else_expr)
+            orel += orel.expr_as_stmt()
 
         body = self._compile_branch(body)
         body += body.expr_as_stmt()

From 65e620ed558fcad2f5527e377630becca986e9bf Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 30 May 2018 10:48:28 -0700
Subject: [PATCH 006/223] Remove an obsolete bug workaround in a test

---
 tests/native_tests/language.hy | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 7e44304..1aa6bef 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -173,8 +173,7 @@
   "Don't treat strings as symbols in the calling position"
   (with [(pytest.raises TypeError)] ("setv" True 3))  ; A special form
   (with [(pytest.raises TypeError)] ("abs" -2))       ; A function
-  (with [(pytest.raises TypeError)] ("when" 1 2))     ; A macro
-  None)  ; Avoid https://github.com/hylang/hy/issues/1320
+  (with [(pytest.raises TypeError)] ("when" 1 2)))    ; A macro
 
 
 (defn test-for-loop []

From 498a54e770610c1af720a91ffaec0e46669a4b02 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 30 May 2018 10:42:00 -0700
Subject: [PATCH 007/223] Fix discovery of tests with mangled names

---
 setup.cfg | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 18c8fbc..e033c3b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,9 +17,8 @@ exclude_lines =
 ignore_errors = True
 
 [tool:pytest]
-# Be sure to include Hy test functions whose names end with "?",
-# which will be mangled to begin with "is_".
-python_functions=test_* is_test_*
+# Be sure to include Hy test functions with mangled names.
+python_functions=test_* is_test_* hyx_test_* hyx_is_test_*
 filterwarnings =
     once::DeprecationWarning
     once::PendingDeprecationWarning

From d621d7c3ab680f5a36915e1f28fad642d11f9883 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 5 Jun 2018 17:08:13 -0700
Subject: [PATCH 008/223] Update defmacro(/g)! tests for mangling

---
 tests/native_tests/native_macros.hy | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index afacee9..e4400ac 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -194,7 +194,7 @@
   (assert (in (mangle "_;a|") s2))
   (assert (not (= s1 s2))))
 
-(defn test-defmacro-g! []
+(defn test-defmacro/g! []
   (import ast)
   (import [astor.code-gen [to-source]])
   (import [hy.importer [import_buffer_to_ast]])
@@ -213,8 +213,8 @@
   (setv _ast2 (import_buffer_to_ast macro1 "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
-  (assert (in "_;res|" s1))
-  (assert (in "_;res|" s2))
+  (assert (in (mangle "_;res|") s1))
+  (assert (in (mangle "_;res|") s2))
   (assert (not (= s1 s2)))
 
   ;; defmacro/g! didn't like numbers initially because they
@@ -242,8 +242,8 @@
   (setv _ast2 (import_buffer_to_ast macro1 "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
-  (assert (in "_;res|" s1))
-  (assert (in "_;res|" s2))
+  (assert (in (mangle "_;res|") s1))
+  (assert (in (mangle "_;res|") s2))
   (assert (not (= s1 s2)))
 
   ;; defmacro/g! didn't like numbers initially because they

From 844256b99bb78bca7e454e4919aa70f5514f6097 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 3 Oct 2017 14:47:35 -0700
Subject: [PATCH 009/223] Make Asty use static rather than instance methods

This ensures `asty.Pass is asty.Pass`.
---
 hy/compiler.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index fe47c9b..bc743e4 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -111,13 +111,13 @@ def builds_model(*model_types):
 # ast.Foo(..., lineno=x.lineno, col_offset=x.col_offset)
 class Asty(object):
     def __getattr__(self, name):
-        setattr(Asty, name, lambda self, x, **kwargs: getattr(ast, name)(
+        setattr(Asty, name, staticmethod(lambda x, **kwargs: getattr(ast, name)(
             lineno=getattr(
                 x, 'start_line', getattr(x, 'lineno', None)),
             col_offset=getattr(
                 x, 'start_column', getattr(x, 'col_offset', None)),
-            **kwargs))
-        return getattr(self, name)
+            **kwargs)))
+        return getattr(Asty, name)
 asty = Asty()
 
 

From 5ffbb4b0eb047ec487ca551da8e4502fd6a38aef Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 3 Oct 2017 14:50:05 -0700
Subject: [PATCH 010/223] Add Result.lineno and Result.col_offset

---
 hy/compiler.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/hy/compiler.py b/hy/compiler.py
index bc743e4..348f80f 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -180,6 +180,22 @@ class Result(object):
         self.__used_expr = False
         self._expr = value
 
+    @property
+    def lineno(self):
+        if self._expr is not None:
+            return self._expr.lineno
+        if self.stmts:
+            return self.stmts[-1].lineno
+        return None
+
+    @property
+    def col_offset(self):
+        if self._expr is not None:
+            return self._expr.col_offset
+        if self.stmts:
+            return self.stmts[-1].col_offset
+        return None
+
     def add_imports(self, mod, imports):
         """Autoimport `imports` from `mod`"""
         self.imports[mod].update(imports)

From 7a40561db8dec676828fbd56afe7407b6f1ebac9 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 29 May 2018 14:36:10 -0700
Subject: [PATCH 011/223] Add tagged model patterns

---
 hy/model_patterns.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/hy/model_patterns.py b/hy/model_patterns.py
index 36b91ae..1d30ccf 100644
--- a/hy/model_patterns.py
+++ b/hy/model_patterns.py
@@ -9,6 +9,7 @@ from funcparserlib.parser import (
     some, skip, many, finished, a, Parser, NoParseError, State)
 from functools import reduce
 from itertools import repeat
+from collections import namedtuple
 from operator import add
 from math import isinf
 
@@ -74,3 +75,11 @@ def times(lo, hi, parser):
             end = e.state.max
         return result, State(s.pos, end)
     return f
+
+Tag = namedtuple('Tag', ['tag', 'value'])
+
+def tag(tag_name, parser):
+    """Matches the given parser and produces a named tuple `(Tag tag value)`
+    with `tag` set to the given tag name and `value` set to the parser's
+    value."""
+    return parser >> (lambda x: Tag(tag_name, x))

From ba1dc55e96834b26f77895f8c9aa1031f84d9a72 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 12 Jun 2018 10:43:17 -0700
Subject: [PATCH 012/223] Implement `lfor`, `sfor`, `gfor`, `dfor`

---
 hy/compiler.py                       | 112 ++++++++++++++++++++++++++-
 tests/native_tests/comprehensions.hy | 106 +++++++++++++++++++++++++
 2 files changed, 217 insertions(+), 1 deletion(-)
 create mode 100644 tests/native_tests/comprehensions.hy

diff --git a/hy/compiler.py b/hy/compiler.py
index 348f80f..82be63b 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -7,7 +7,7 @@ from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
                        HyString, HyBytes, HySymbol, HyFloat, HyList, HySet,
                        HyDict, HySequence, wrap_value)
 from hy.model_patterns import (FORM, SYM, STR, sym, brackets, whole, notpexpr,
-                               dolike, pexpr, times)
+                               dolike, pexpr, times, Tag, tag)
 from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
 from hy.errors import HyCompileError, HyTypeError
 
@@ -1055,6 +1055,116 @@ class HyASTCompiler(object):
             value=value.force_expr,
             generators=gen)
 
+    _loopers = many(
+        tag('setv', sym(":setv") + FORM + FORM) |
+        tag('if', sym(":if") + FORM) |
+        tag('do', sym(":do") + FORM) |
+        tag('afor', sym(":async") + FORM + FORM) |
+        tag('for', FORM + FORM))
+    @special(["lfor", "sfor", "gfor"], [_loopers, FORM])
+    @special(["dfor"], [_loopers, brackets(FORM, FORM)])
+    def compile_new_comp(self, expr, root, parts, final):
+        node_class = dict(
+            lfor=asty.ListComp,
+            dfor=asty.DictComp,
+            sfor=asty.SetComp,
+            gfor=asty.GeneratorExp)[str(root)]
+
+        # Compile the final value (and for dictionary comprehensions, the final
+        # key).
+        if node_class is asty.DictComp:
+            key, elt = map(self.compile, final)
+        else:
+            key = None
+            elt = self.compile(final)
+
+        # Compile the parts.
+        parts = [
+            Tag(p.tag, self.compile(p.value) if p.tag in ["if", "do"] else [
+                self._storeize(p.value[0], self.compile(p.value[0])),
+                self.compile(p.value[1])])
+            for p in parts]
+
+        # Produce a result.
+        if (elt.stmts or (key is not None and key.stmts) or
+            any(p.tag == 'do' or (p.value[1].stmts if p.tag in ("for", "afor", "setv") else p.value.stmts)
+                for p in parts)):
+            # The desired comprehension can't be expressed as a
+            # real Python comprehension. We'll write it as a nested
+            # loop in a function instead.
+            def f(parts):
+                # This function is called recursively to construct
+                # the nested loop.
+                if not parts:
+                    if node_class is asty.DictComp:
+                        ret = key + elt
+                        val = asty.Tuple(
+                            key, ctx=ast.Load(),
+                            elts=[key.force_expr, elt.force_expr])
+                    else:
+                        ret = elt
+                        val = elt.force_expr
+                    return ret + asty.Expr(
+                        elt, value=asty.Yield(elt, value=val))
+                (tagname, v), parts = parts[0], parts[1:]
+                if tagname in ("for", "afor"):
+                    node = asty.AsyncFor if tagname == "afor" else asty.For
+                    return v[1] + node(
+                        v[1], target=v[0], iter=v[1].force_expr, body=f(parts).stmts,
+                        orelse=[])
+                elif tagname == "setv":
+                    return v[1] + asty.Assign(
+                        v[1], targets=[v[0]], value=v[1].force_expr) + f(parts)
+                elif tagname == "if":
+                    return v + asty.If(
+                        v, test=v.force_expr, body=f(parts).stmts, orelse=[])
+                elif tagname == "do":
+                    return v + v.expr_as_stmt() + f(parts)
+                else:
+                    raise ValueError("can't happen")
+            fname = self.get_anon_var()
+            # Define the generator function.
+            ret = Result() + asty.FunctionDef(
+                expr,
+                name=fname,
+                args=ast.arguments(
+                    args=[], vararg=None, kwarg=None,
+                    kwonlyargs=[], kw_defaults=[], defaults=[]),
+                body=f(parts).stmts,
+                decorator_list=[])
+            # Immediately call the new function. Unless the user asked
+            # for a generator, wrap the call in `[].__class__(...)` or
+            # `{}.__class__(...)` or `{1}.__class__(...)` to get the
+            # right type. We don't want to just use e.g. `list(...)`
+            # because the name `list` might be rebound.
+            return ret + Result(expr=ast.parse(
+                "{}({}())".format(
+                    {asty.ListComp: "[].__class__",
+                     asty.DictComp: "{}.__class__",
+                     asty.SetComp: "{1}.__class__",
+                     asty.GeneratorExp: ""}[node_class],
+                    fname)).body[0].value)
+
+        # We can produce a real comprehension.
+        generators = []
+        for tagname, v in parts:
+            if tagname in ("for", "afor"):
+                generators.append(ast.comprehension(
+                    target=v[0], iter=v[1].expr, ifs=[],
+                    is_async=int(tagname == "afor")))
+            elif tagname == "setv":
+                generators.append(ast.comprehension(
+                    target=v[0],
+                    iter=asty.Tuple(v[1], elts=[v[1].expr], ctx=ast.Load()),
+                    ifs=[], is_async=0))
+            elif tagname == "if":
+                generators[-1].ifs.append(v.expr)
+            else:
+                raise ValueError("can't happen")
+        if node_class is asty.DictComp:
+            return asty.DictComp(expr, key=key.expr, value=elt.expr, generators=generators)
+        return node_class(expr, elt=elt.expr, generators=generators)
+
     @special(["not", "~"], [FORM])
     def compile_unary_operator(self, expr, root, arg):
         ops = {"not": ast.Not,
diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy
new file mode 100644
index 0000000..9c1a264
--- /dev/null
+++ b/tests/native_tests/comprehensions.hy
@@ -0,0 +1,106 @@
+(import
+  types
+  pytest)
+
+
+(defn test-comprehension-types []
+
+  ; Forms that get compiled to real comprehensions
+  (assert (is (type (lfor x "abc" x)) list))
+  (assert (is (type (sfor x "abc" x)) set))
+  (assert (is (type (dfor x "abc" [x x])) dict))
+  (assert (is (type (gfor x "abc" x)) types.GeneratorType))
+
+  ; Forms that get compiled to loops
+  (assert (is (type (lfor x "abc" :do (setv y 1) x)) list))
+  (assert (is (type (sfor x "abc" :do (setv y 1) x)) set))
+  (assert (is (type (dfor x "abc" :do (setv y 1) [x x])) dict))
+  (assert (is (type (gfor x "abc" :do (setv y 1) x)) types.GeneratorType)))
+
+
+#@ ((pytest.mark.parametrize "specialop" ["lfor" "sfor" "gfor" "dfor"])
+(defn test-comprehensions [specialop]
+
+  (setv cases [
+    ['(f x [] x)
+      []]
+    ['(f j [1 2 3] j)
+      [1 2 3]]
+    ['(f x (range 3) (* x 2))
+      [0 2 4]]
+    ['(f x (range 2) y (range 2) (, x y))
+      [(, 0 0) (, 0 1) (, 1 0) (, 1 1)]]
+    ['(f (, x y) (.items {"1" 1 "2" 2}) (* y 2))
+      [2 4]]
+    ['(f x (do (setv s "x") "ab") y (do (+= s "y") "def") (+ x y s))
+      ["adxy" "aexy" "afxy" "bdxyy" "bexyy" "bfxyy"]]
+    ['(f x (range 4) :if (% x 2) (* x 2))
+      [2 6]]
+    ['(f x "abc" :setv y (.upper x) (+ x y))
+      ["aA" "bB" "cC"]]
+    ['(f x "abc" :do (setv y (.upper x)) (+ x y))
+      ["aA" "bB" "cC"]]
+    ['(f
+        x (range 3)
+        y (range 3)
+        :if (> y x)
+        z [7 8 9]
+        :setv s (+ x y z)
+        :if (!= z 8)
+        (, x y z s))
+      [(, 0 1 7 8) (, 0 1 9 10) (, 0 2 7 9) (, 0 2 9 11)
+        (, 1 2 7 10) (, 1 2 9 12)]]
+    ['(f
+        x [0 1]
+        :setv l []
+        y (range 4)
+        :do (.append l (, x y))
+        :if (>= y 2)
+        z [7 8 9]
+        :if (!= z 8)
+        (, x y (tuple l) z))
+      [(, 0 2 (, (, 0 0) (, 0 1) (, 0 2)) 7)
+        (, 0 2 (, (, 0 0) (, 0 1) (, 0 2)) 9)
+        (, 0 3 (, (, 0 0) (, 0 1) (, 0 2) (, 0 3)) 7)
+        (, 0 3 (, (, 0 0) (, 0 1) (, 0 2) (, 0 3)) 9)
+        (, 1 2 (, (, 1 0) (, 1 1) (, 1 2)) 7)
+        (, 1 2 (, (, 1 0) (, 1 1) (, 1 2)) 9)
+        (, 1 3 (, (, 1 0) (, 1 1) (, 1 2) (, 1 3)) 7)
+        (, 1 3 (, (, 1 0) (, 1 1) (, 1 2) (, 1 3)) 9)]]
+
+    ['(f x (range 4) :do (unless (% x 2) (continue)) (* x 2))
+      [2 6]]
+    ['(f x (range 4) :setv p 9 :do (unless (% x 2) (continue)) (* x 2))
+      [2 6]]
+    ['(f x (range 20) :do (when (= x 3) (break)) (* x 2))
+      [0 2 4]]
+    ['(f x (range 20) :setv p 9 :do (when (= x 3) (break)) (* x 2))
+      [0 2 4]]
+    ['(f x [4 5] y (range 20) :do (when (> y 1) (break)) z [8 9] (, x y z))
+      [(, 4 0 8) (, 4 0 9) (, 4 1 8) (, 4 1 9)
+        (, 5 0 8) (, 5 0 9) (, 5 1 8) (, 5 1 9)]]])
+
+  (for [[expr answer] cases]
+    ; Mutate the case as appropriate for the operator before
+    ; evaluating it.
+    (setv expr (+ (HyExpression [(HySymbol specialop)]) (cut expr 1)))
+    (when (= specialop "dfor")
+      (setv expr (+ (cut expr 0 -1) `([~(get expr -1) 1]))))
+    (setv result (eval expr))
+    (when (= specialop "dfor")
+      (setv result (.keys result)))
+    (assert (= (sorted result) answer) (str expr)))))
+
+
+(defn test-raise-in-comp []
+  (defclass E [Exception] [])
+  (setv l [])
+  (import pytest)
+  (with [(pytest.raises E)]
+    (lfor
+      x (range 10)
+      :do (.append l x)
+      :do (when (= x 5)
+        (raise (E)))
+      x))
+  (assert (= l [0 1 2 3 4 5])))

From 3256932b13afdc67e53a48c9699e5460edf90c5e Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 12 Jun 2018 10:54:08 -0700
Subject: [PATCH 013/223] Add a version of `for` parallel to `lfor` etc.

---
 hy/compiler.py                       |  54 ++++++++++----
 hy/core/macros.hy                    |   9 +--
 tests/compilers/test_ast.py          |   4 +-
 tests/native_tests/comprehensions.hy |  87 +++++++++++++++++++++-
 tests/native_tests/contrib/walk.hy   |   2 +-
 tests/native_tests/language.hy       | 104 ---------------------------
 6 files changed, 129 insertions(+), 131 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 82be63b..fb47f4b 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1061,24 +1061,39 @@ class HyASTCompiler(object):
         tag('do', sym(":do") + FORM) |
         tag('afor', sym(":async") + FORM + FORM) |
         tag('for', FORM + FORM))
+    @special(["for"], [brackets(_loopers),
+        many(notpexpr("else")) + maybe(dolike("else"))])
     @special(["lfor", "sfor", "gfor"], [_loopers, FORM])
     @special(["dfor"], [_loopers, brackets(FORM, FORM)])
     def compile_new_comp(self, expr, root, parts, final):
-        node_class = dict(
-            lfor=asty.ListComp,
-            dfor=asty.DictComp,
-            sfor=asty.SetComp,
-            gfor=asty.GeneratorExp)[str(root)]
+        root = unmangle(ast_str(root))
+        node_class = {
+            "for":  asty.For,
+            "lfor": asty.ListComp,
+            "dfor": asty.DictComp,
+            "sfor": asty.SetComp,
+            "gfor": asty.GeneratorExp}[root]
+        is_for = root == "for"
 
-        # Compile the final value (and for dictionary comprehensions, the final
-        # key).
-        if node_class is asty.DictComp:
-            key, elt = map(self.compile, final)
+        orel = []
+        if is_for:
+            # Get the `else`.
+            body, else_expr = final
+            if else_expr is not None:
+                orel.append(self._compile_branch(else_expr))
+                orel[0] += orel[0].expr_as_stmt()
         else:
-            key = None
-            elt = self.compile(final)
+            # Get the final value (and for dictionary
+            # comprehensions, the final key).
+            if node_class is asty.DictComp:
+                key, elt = map(self.compile, final)
+            else:
+                key = None
+                elt = self.compile(final)
 
         # Compile the parts.
+        if is_for:
+            parts = parts[0]
         parts = [
             Tag(p.tag, self.compile(p.value) if p.tag in ["if", "do"] else [
                 self._storeize(p.value[0], self.compile(p.value[0])),
@@ -1086,16 +1101,24 @@ class HyASTCompiler(object):
             for p in parts]
 
         # Produce a result.
-        if (elt.stmts or (key is not None and key.stmts) or
+        if (is_for or elt.stmts or (key is not None and key.stmts) or
             any(p.tag == 'do' or (p.value[1].stmts if p.tag in ("for", "afor", "setv") else p.value.stmts)
                 for p in parts)):
             # The desired comprehension can't be expressed as a
             # real Python comprehension. We'll write it as a nested
             # loop in a function instead.
+            contains_yield = []
             def f(parts):
                 # This function is called recursively to construct
                 # the nested loop.
                 if not parts:
+                    if is_for:
+                        if body:
+                            bd = self._compile_branch(body)
+                            if bd.contains_yield:
+                                contains_yield.append(True)
+                            return bd + bd.expr_as_stmt()
+                        return Result(stmts=[asty.Pass(expr)])
                     if node_class is asty.DictComp:
                         ret = key + elt
                         val = asty.Tuple(
@@ -1108,10 +1131,11 @@ class HyASTCompiler(object):
                         elt, value=asty.Yield(elt, value=val))
                 (tagname, v), parts = parts[0], parts[1:]
                 if tagname in ("for", "afor"):
+                    orelse = orel and orel.pop().stmts
                     node = asty.AsyncFor if tagname == "afor" else asty.For
                     return v[1] + node(
                         v[1], target=v[0], iter=v[1].force_expr, body=f(parts).stmts,
-                        orelse=[])
+                        orelse=orelse)
                 elif tagname == "setv":
                     return v[1] + asty.Assign(
                         v[1], targets=[v[0]], value=v[1].force_expr) + f(parts)
@@ -1122,6 +1146,10 @@ class HyASTCompiler(object):
                     return v + v.expr_as_stmt() + f(parts)
                 else:
                     raise ValueError("can't happen")
+            if is_for:
+                ret = f(parts)
+                ret.contains_yield = bool(contains_yield)
+                return ret
             fname = self.get_anon_var()
             # Define the generator function.
             ret = Result() + asty.FunctionDef(
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index a948fd8..b87f110 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -122,14 +122,6 @@ used as the result."
       `(~node [(, ~@alist) (genexpr (, ~@alist) [~@args])] (do ~@body) ~@belse))))
 
 
-(defmacro for [args &rest body]
-  "Build a for-loop with `args` as a [element coll] bracket pair and run `body`.
-
-Args may contain multiple pairs, in which case it executes a nested for-loop
-in order of the given pairs."
-  (_for 'for* args body))
-
-
 (defmacro for/a [args &rest body]
   "Build a for/a-loop with `args` as a [element coll] bracket pair and run `body`.
 
@@ -163,6 +155,7 @@ the second form, the second result is inserted into the third form, and so on."
      ~@(map build-form expressions)
      ~f))
 
+
 (defmacro ->> [head &rest args]
   "Thread `head` last through the `rest` of the forms.
 
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index d232649..ad41b01 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -531,9 +531,7 @@ def test_for_compile_error():
         can_compile("(fn [] (for)))")
     assert excinfo.value.message == "Ran into a RPAREN where it wasn't expected."
 
-    with pytest.raises(HyTypeError) as excinfo:
-        can_compile("(fn [] (for [x] x))")
-    assert excinfo.value.message == "`for' requires an even number of args."
+    cant_compile("(fn [] (for [x] x))")
 
 
 def test_attribute_access():
diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy
index 9c1a264..b450e2f 100644
--- a/tests/native_tests/comprehensions.hy
+++ b/tests/native_tests/comprehensions.hy
@@ -18,8 +18,8 @@
   (assert (is (type (gfor x "abc" :do (setv y 1) x)) types.GeneratorType)))
 
 
-#@ ((pytest.mark.parametrize "specialop" ["lfor" "sfor" "gfor" "dfor"])
-(defn test-comprehensions [specialop]
+#@ ((pytest.mark.parametrize "specialop" ["for" "lfor" "sfor" "gfor" "dfor"])
+(defn test-fors [specialop]
 
   (setv cases [
     ['(f x [] x)
@@ -86,6 +86,12 @@
     (setv expr (+ (HyExpression [(HySymbol specialop)]) (cut expr 1)))
     (when (= specialop "dfor")
       (setv expr (+ (cut expr 0 -1) `([~(get expr -1) 1]))))
+    (when (= specialop "for")
+      (setv expr `(do
+        (setv out [])
+        (for [~@(cut expr 1 -1)]
+          (.append out ~(get expr -1)))
+        out)))
     (setv result (eval expr))
     (when (= specialop "dfor")
       (setv result (.keys result)))
@@ -104,3 +110,80 @@
         (raise (E)))
       x))
   (assert (= l [0 1 2 3 4 5])))
+
+
+(defn test-for-loop []
+  "NATIVE: test for loops"
+  (setv count1 0 count2 0)
+  (for [x [1 2 3 4 5]]
+    (setv count1 (+ count1 x))
+    (setv count2 (+ count2 x)))
+  (assert (= count1 15))
+  (assert (= count2 15))
+  (setv count 0)
+  (for [x [1 2 3 4 5]
+        y [1 2 3 4 5]]
+    (setv count (+ count x y))
+    (else
+      (+= count 1)))
+  (assert (= count 151))
+
+  (setv count 0)
+  ; multiple statements in the else branch should work
+  (for [x [1 2 3 4 5]
+        y [1 2 3 4 5]]
+    (setv count (+ count x y))
+    (else
+      (+= count 1)
+      (+= count 10)))
+  (assert (= count 161))
+
+  ; don't be fooled by constructs that look like else
+  (setv s "")
+  (setv else True)
+  (for [x "abcde"]
+    (+= s x)
+    [else (+= s "_")])
+  (assert (= s "a_b_c_d_e_"))
+
+  (setv s "")
+  (with [(pytest.raises TypeError)]
+    (for [x "abcde"]
+      (+= s x)
+      ("else" (+= s "z"))))
+  (assert (= s "az"))
+
+  (assert (= (list ((fn [] (for [x [[1] [2 3]] y x] (yield y)))))
+             (list-comp y [x [[1] [2 3]] y x])))
+  (assert (= (list ((fn [] (for [x [[1] [2 3]] y x z (range 5)] (yield z)))))
+             (list-comp z [x [[1] [2 3]] y x z (range 5)]))))
+
+
+(defn test-nasty-for-nesting []
+  "NATIVE: test nesting for loops harder"
+  ;; This test and feature is dedicated to @nedbat.
+
+  ;; OK. This next test will ensure that we call the else branch exactly
+  ;; once.
+  (setv flag 0)
+  (for [x (range 2)
+        y (range 2)]
+    (+ 1 1)
+    (else (setv flag (+ flag 2))))
+  (assert (= flag 2)))
+
+
+(defn test-empty-for []
+
+  (setv l [])
+  (defn f []
+    (for [x (range 3)]
+      (.append l "a")
+      (yield x)))
+  (for [x (f)])
+  (assert (= l ["a" "a" "a"]))
+
+  (setv l [])
+  (for [x (f)]
+    (else (.append l "z")))
+  (assert (= l ["a" "a" "a" "z"])))
diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index e2d7a2a..7facd59 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -46,7 +46,7 @@
   (assert (= (macroexpand-all '(with [a 1]))
              '(with* [a 1] (do))))
   (assert (= (macroexpand-all '(with [a 1 b 2 c 3] (for [d c] foo)))
-             '(with* [a 1] (with* [b 2] (with* [c 3] (do (for* [d c] (do foo))))))))
+             '(with* [a 1] (with* [b 2] (with* [c 3] (do (for [d c] foo)))))))
   (assert (= (macroexpand-all '(with [a 1]
                                  '(with [b 2])
                                  `(with [c 3]
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 1aa6bef..8dd978b 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -176,110 +176,6 @@
   (with [(pytest.raises TypeError)] ("when" 1 2)))    ; A macro
 
 
-(defn test-for-loop []
-  "NATIVE: test for loops"
-  (setv count1 0 count2 0)
-  (for [x [1 2 3 4 5]]
-    (setv count1 (+ count1 x))
-    (setv count2 (+ count2 x)))
-  (assert (= count1 15))
-  (assert (= count2 15))
-  (setv count 0)
-  (for [x [1 2 3 4 5]
-        y [1 2 3 4 5]]
-    (setv count (+ count x y))
-    (else
-      (+= count 1)))
-  (assert (= count 151))
-
-  (setv count 0)
-  ; multiple statements in the else branch should work
-  (for [x [1 2 3 4 5]
-        y [1 2 3 4 5]]
-    (setv count (+ count x y))
-    (else
-      (+= count 1)
-      (+= count 10)))
-  (assert (= count 161))
-
-  ; don't be fooled by constructs that look like else
-  (setv s "")
-  (setv else True)
-  (for [x "abcde"]
-    (+= s x)
-    [else (+= s "_")])
-  (assert (= s "a_b_c_d_e_"))
-
-  (setv s "")
-  (setv else True)
-  (with [(pytest.raises TypeError)]
-    (for [x "abcde"]
-      (+= s x)
-      ("else" (+= s "z"))))
-  (assert (= s "az"))
-
-  (assert (= (list ((fn [] (for [x [[1] [2 3]] y x] (yield y)))))
-             (list-comp y [x [[1] [2 3]] y x])))
-  (assert (= (list ((fn [] (for [x [[1] [2 3]] y x z (range 5)] (yield z)))))
-             (list-comp z [x [[1] [2 3]] y x z (range 5)])))
-
-  (setv l [])
-  (defn f []
-    (for [x [4 9 2]]
-      (.append l (* 10 x))
-      (yield x)))
-  (for [_ (f)])
-  (assert (= l [40 90 20])))
-
-
-(defn test-nasty-for-nesting []
-  "NATIVE: test nesting for loops harder"
-  ;; This test and feature is dedicated to @nedbat.
-
-  ;; let's ensure empty iterating is an implicit do
-  (setv t 0)
-  (for [] (setv t 1))
-  (assert (= t 1))
-
-  ;; OK. This first test will ensure that the else is hooked up to the
-  ;; for when we break out of it.
-  (for [x (range 2)
-        y (range 2)]
-      (break)
-    (else (raise Exception)))
-
-  ;; OK. This next test will ensure that the else is hooked up to the
-  ;; "inner" iteration
-  (for [x (range 2)
-        y (range 2)]
-    (if (= y 1) (break))
-    (else (raise Exception)))
-
-  ;; OK. This next test will ensure that the else is hooked up to the
-  ;; "outer" iteration
-  (for [x (range 2)
-        y (range 2)]
-    (if (= x 1) (break))
-    (else (raise Exception)))
-
-  ;; OK. This next test will ensure that we call the else branch exactly
-  ;; once.
-  (setv flag 0)
-  (for [x (range 2)
-        y (range 2)]
-    (+ 1 1)
-    (else (setv flag (+ flag 2))))
-  (assert (= flag 2))
-
-  (setv l [])
-  (defn f []
-    (for [x [4 9 2]]
-      (.append l (* 10 x))
-      (yield x)))
-  (for [_ (f)])
-  (assert (= l [40 90 20])))
-
-
 (defn test-while-loop []
   "NATIVE: test while loops?"
   (setv count 5)

From e1972c535fac571fc6609d7f032f4e9e1c0d5ff0 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 12 Jun 2018 10:57:57 -0700
Subject: [PATCH 014/223] Remove `for/a`, `for*`, and `for/a*`

---
 hy/compiler.py                        | 30 -------------------------
 hy/core/bootstrap.hy                  |  2 +-
 hy/core/language.hy                   | 12 +++++-----
 hy/core/macros.hy                     | 32 +++++----------------------
 hy/core/shadow.hy                     |  2 +-
 tests/native_tests/language.hy        |  4 ++--
 tests/native_tests/native_macros.hy   |  4 ++--
 tests/native_tests/py36_only_tests.hy |  8 +++----
 tests/native_tests/py3_only_tests.hy  |  4 ++--
 9 files changed, 23 insertions(+), 75 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index fb47f4b..cbe0e30 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1442,36 +1442,6 @@ class HyASTCompiler(object):
 
         return result
 
-    @special(["for*", (PY35, "for/a*")],
-             [brackets(FORM, FORM), many(notpexpr("else")), maybe(dolike("else"))])
-    def compile_for_expression(self, expr, root, args, body, else_expr):
-        target_name, iterable = args
-        target = self._storeize(target_name, self.compile(target_name))
-
-        ret = Result()
-
-        orel = Result()
-        if else_expr is not None:
-            for else_body in else_expr:
-                orel += self.compile(else_body)
-                orel += orel.expr_as_stmt()
-
-        ret += self.compile(iterable)
-
-        body = self._compile_branch(body)
-        body += body.expr_as_stmt()
-
-        node = asty.For if root == 'for*' else asty.AsyncFor
-        ret += node(expr,
-                    target=target,
-                    iter=ret.force_expr,
-                    body=body.stmts or [asty.Pass(expr)],
-                    orelse=orel.stmts)
-
-        ret.contains_yield = body.contains_yield
-
-        return ret
-
     @special(["while"], [FORM, many(notpexpr("else")), maybe(dolike("else"))])
     def compile_while_expression(self, expr, root, cond, body, else_expr):
         cond_compiled = self.compile(cond)
diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy
index 14539c3..cc576bc 100644
--- a/hy/core/bootstrap.hy
+++ b/hy/core/bootstrap.hy
@@ -18,7 +18,7 @@
               (% "received a `%s' instead of a symbol for macro name"
                  (. (type name)
                     __name__)))))
-     (for* [kw '[&kwonly &kwargs]]
+     (for [kw '[&kwonly &kwargs]]
        (if* (in kw lambda-list)
             (raise (hy.errors.HyTypeError macro-name
                                           (% "macros cannot use %s"
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 3481bf1..3f0dd39 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -41,7 +41,7 @@
                 fs (tuple rfs))
           (fn [&rest args &kwargs kwargs]
             (setv res (first-f #* args #** kwargs))
-            (for* [f fs]
+            (for [f fs]
               (setv res (f res)))
             res))))
 
@@ -79,7 +79,7 @@ If the second argument `codegen` is true, generate python code instead."
 (defn distinct [coll]
   "Return a generator from the original collection `coll` with no duplicates."
   (setv seen (set) citer (iter coll))
-  (for* [val citer]
+  (for [val citer]
     (if (not-in val seen)
       (do
        (yield val)
@@ -159,7 +159,7 @@ Return series of accumulated sums (or other binary function results)."
   (setv it (iter iterable)
         total (next it))
   (yield total)
-  (for* [element it]
+  (for [element it]
     (setv total (func total element))
     (yield total)))
 
@@ -193,7 +193,7 @@ Return series of accumulated sums (or other binary function results)."
 
 (defn _flatten [coll result]
   (if (coll? coll)
-    (do (for* [b coll]
+    (do (for [b coll]
           (_flatten b result)))
     (.append result coll))
   result)
@@ -402,9 +402,9 @@ Raises ValueError for (not (pos? n))."
   (if (not (pos? n))
     (raise (ValueError "n must be positive")))
   (setv citer (iter coll) skip (dec n))
-  (for* [val citer]
+  (for [val citer]
     (yield val)
-    (for* [_ (range skip)]
+    (for [_ (range skip)]
       (try
         (next citer)
         (except [StopIteration]
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index b87f110..05e67e9 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -101,42 +101,20 @@ used as the result."
      (setv root (check-branch branch))
      (setv latest-branch root)
 
-     (for* [branch branches]
+     (for [branch branches]
        (setv cur-branch (check-branch branch))
        (.append latest-branch cur-branch)
        (setv latest-branch cur-branch))
      root)))
 
 
-(defn _for [node args body]
-  (setv body (list body))
-  (setv belse (if (and body (isinstance (get body -1) HyExpression) (= (get body -1 0) "else"))
-                [(body.pop)]
-                []))
-  (if
-    (odd? (len args)) (macro-error args "`for' requires an even number of args.")
-    (empty? args)     `(do ~@body ~@belse)
-    (= (len args) 2)  `(~node [~@args] (do ~@body) ~@belse)
-    (do
-      (setv alist (cut args 0 None 2))
-      `(~node [(, ~@alist) (genexpr (, ~@alist) [~@args])] (do ~@body) ~@belse))))
-
-
-(defmacro for/a [args &rest body]
-  "Build a for/a-loop with `args` as a [element coll] bracket pair and run `body`.
-
-Args may contain multiple pairs, in which case it executes a nested for/a-loop
-in order of the given pairs."
-  (_for 'for/a* args body))
-
-
 (defmacro -> [head &rest args]
   "Thread `head` first through the `rest` of the forms.
 
 The result of the first threaded form is inserted into the first position of
 the second form, the second result is inserted into the third form, and so on."
   (setv ret head)
-  (for* [node args]
+  (for [node args]
     (setv ret (if (isinstance node HyExpression)
                   `(~(first node) ~ret ~@(rest node))
                   `(~node ~ret))))
@@ -162,7 +140,7 @@ the second form, the second result is inserted into the third form, and so on."
 The result of the first threaded form is inserted into the last position of
 the second form, the second result is inserted into the third form, and so on."
   (setv ret head)
-  (for* [node args]
+  (for [node args]
     (setv ret (if (isinstance node HyExpression)
                   `(~@node ~ret)
                   `(~node ~ret))))
@@ -203,7 +181,7 @@ the second form, the second result is inserted into the third form, and so on."
 (defmacro with-gensyms [args &rest body]
   "Execute `body` with `args` as bracket of names to gensym for use in macros."
   (setv syms [])
-  (for* [arg args]
+  (for [arg args]
     (.extend syms [arg `(gensym '~arg)]))
   `(do
     (setv ~@syms)
@@ -218,7 +196,7 @@ the second form, the second result is inserted into the third form, and so on."
                               (.startswith x "g!")))
                        (flatten body))))
         gensyms [])
-  (for* [sym syms]
+  (for [sym syms]
     (.extend gensyms [sym `(gensym ~(cut sym 2))]))
   `(defmacro ~name [~@args]
      (setv ~@gensyms)
diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy
index c69f344..1f75108 100644
--- a/hy/core/shadow.hy
+++ b/hy/core/shadow.hy
@@ -161,7 +161,7 @@
 (defn get [coll key1 &rest keys]
   "Access item in `coll` indexed by `key1`, with optional `keys` nested-access."
   (setv coll (get coll key1))
-  (for* [k keys]
+  (for [k keys]
     (setv coll (get coll k)))
   coll)
 
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 8dd978b..9ce8da2 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -830,13 +830,13 @@
 (defn test-for-else []
   "NATIVE: test for else"
   (setv x 0)
-  (for* [a [1 2]]
+  (for [a [1 2]]
     (setv x (+ x a))
     (else (setv x (+ x 50))))
   (assert (= x 53))
 
   (setv x 0)
-  (for* [a [1 2]]
+  (for [a [1 2]]
     (setv x (+ x a))
     (else))
   (assert (= x 3)))
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index e4400ac..79ae560 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -101,7 +101,7 @@
 (defn test-midtree-yield-in-for []
   "NATIVE: test yielding in a for with a return"
   (defn kruft-in-for []
-    (for* [i (range 5)]
+    (for [i (range 5)]
       (yield i))
     (+ 1 2)))
 
@@ -117,7 +117,7 @@
 (defn test-multi-yield []
   "NATIVE: testing multiple yields"
   (defn multi-yield []
-    (for* [i (range 3)]
+    (for [i (range 3)]
       (yield i))
     (yield "a")
     (yield "end"))
diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy
index 6a71fcf..d324a26 100644
--- a/tests/native_tests/py36_only_tests.hy
+++ b/tests/native_tests/py36_only_tests.hy
@@ -13,7 +13,7 @@
   (.run_until_complete (get-event-loop) (coro)))
 
 
-(defn test-for/a []
+(defn test-for-async []
   (defn/a numbers []
     (for [i [1 2]]
       (yield i)))
@@ -21,11 +21,11 @@
   (run-coroutine
     (fn/a []
       (setv x 0)
-      (for/a [a (numbers)]
+      (for [:async a (numbers)]
         (setv x (+ x a)))
       (assert (= x 3)))))
 
-(defn test-for/a-else []
+(defn test-for-async-else []
   (defn/a numbers []
     (for [i [1 2]]
       (yield i)))
@@ -33,7 +33,7 @@
   (run-coroutine
     (fn/a []
       (setv x 0)
-      (for/a [a (numbers)]
+      (for [:async a (numbers)]
         (setv x (+ x a))
         (else (setv x (+ x 50))))
       (assert (= x 53)))))
diff --git a/tests/native_tests/py3_only_tests.hy b/tests/native_tests/py3_only_tests.hy
index 00cd448..1d69546 100644
--- a/tests/native_tests/py3_only_tests.hy
+++ b/tests/native_tests/py3_only_tests.hy
@@ -49,7 +49,7 @@
 (defn test-yield-from []
   "NATIVE: testing yield from"
   (defn yield-from-test []
-    (for* [i (range 3)]
+    (for [i (range 3)]
       (yield i))
     (yield-from [1 2 3]))
   (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))
@@ -63,7 +63,7 @@
     (yield 3)
     (assert 0))
   (defn yield-from-test []
-    (for* [i (range 3)]
+    (for [i (range 3)]
       (yield i))
     (try
       (yield-from (yield-from-subgenerator-test))

From 4754b152a996549e209207ed3b46bb7861d3a47b Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 12 Jun 2018 11:00:01 -0700
Subject: [PATCH 015/223] Allow comprehensions with no looping parts

---
 hy/compiler.py                       |  7 +++++++
 tests/native_tests/comprehensions.hy | 12 ++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/hy/compiler.py b/hy/compiler.py
index cbe0e30..d6bc53d 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1094,6 +1094,13 @@ class HyASTCompiler(object):
         # Compile the parts.
         if is_for:
             parts = parts[0]
+        if not parts:
+            return Result(expr=ast.parse({
+                asty.For: "None",
+                asty.ListComp: "[]",
+                asty.DictComp: "{}",
+                asty.SetComp: "{1}.__class__()",
+                asty.GeneratorExp: "(_ for _ in [])"}[node_class]).body[0].value)
         parts = [
             Tag(p.tag, self.compile(p.value) if p.tag in ["if", "do"] else [
                 self._storeize(p.value[0], self.compile(p.value[0])),
diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy
index b450e2f..fcd0719 100644
--- a/tests/native_tests/comprehensions.hy
+++ b/tests/native_tests/comprehensions.hy
@@ -98,6 +98,18 @@
     (assert (= (sorted result) answer) (str expr)))))
 
 
+(defn test-fors-no-loopers []
+
+  (setv l [])
+  (for [] (.append l 1))
+  (assert (= l []))
+
+  (assert (= (lfor 1) []))
+  (assert (= (sfor 1) #{}))
+  (assert (= (list (gfor 1)) []))
+  (assert (= (dfor [1 2]) {})))
+
+
 (defn test-raise-in-comp []
   (defclass E [Exception] [])
   (setv l [])

From df4e49ec94f39a5ad9fc3e2d9e9e03272b686be6 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 7 Jun 2018 11:08:17 -0700
Subject: [PATCH 016/223] Test comprehension scoping

---
 tests/native_tests/comprehensions.hy | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy
index fcd0719..1f17912 100644
--- a/tests/native_tests/comprehensions.hy
+++ b/tests/native_tests/comprehensions.hy
@@ -1,6 +1,7 @@
 (import
   types
-  pytest)
+  pytest
+  [hy._compat [PY3]])
 
 
 (defn test-comprehension-types []
@@ -124,6 +125,30 @@
   (assert (= l [0 1 2 3 4 5])))
 
 
+(defn test-scoping []
+
+  (setv x 0)
+  (for [x [1 2 3]])
+  (assert (= x 3))
+
+  ; An `lfor` that gets compiled to a real comprehension
+  (setv x 0)
+  (assert (= (lfor x [1 2 3] (inc x)) [2 3 4]))
+  (assert (= x (if PY3 0 3)))
+    ; Python 2 list comprehensions leak their variables.
+
+  ; An `lfor` that gets compiled to a loop
+  (setv x 0  l [])
+  (assert (= (lfor x [4 5 6] :do (.append l 1) (inc x)) [5 6 7]))
+  (assert (= l [1 1 1]))
+  (assert (= x 0))
+
+  ; An `sfor` that gets compiled to a real comprehension
+  (setv x 0)
+  (assert (= (sfor x [1 2 3] (inc x)) #{2 3 4}))
+  (assert (= x 0)))
+
+
 (defn test-for-loop []
   "NATIVE: test for loops"
   (setv count1 0 count2 0)

From cf0dafef9bed2fc61ba9ec3e94b5c6ac5ac20bba Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 7 Jun 2018 12:18:49 -0700
Subject: [PATCH 017/223] Update uses of the old comprehension forms

---
 hy/contrib/hy_repr.hy                 | 26 +++++++++++++++-----------
 hy/contrib/sequences.hy               |  4 ++--
 hy/core/language.hy                   |  6 +++---
 hy/core/macros.hy                     | 10 +++++-----
 hy/core/shadow.hy                     |  3 +--
 hy/extra/anaphoric.hy                 | 22 +++++++++++-----------
 tests/importer/test_importer.py       |  2 +-
 tests/native_tests/comprehensions.hy  |  4 ++--
 tests/native_tests/contrib/hy_repr.hy |  6 +++---
 tests/native_tests/tag_macros.hy      |  2 +-
 tests/resources/pydemo.hy             |  8 ++++----
 11 files changed, 48 insertions(+), 45 deletions(-)

diff --git a/hy/contrib/hy_repr.hy b/hy/contrib/hy_repr.hy
index 42ff62c..aca8b2a 100644
--- a/hy/contrib/hy_repr.hy
+++ b/hy/contrib/hy_repr.hy
@@ -25,9 +25,10 @@
 (setv -seen (set))
 (defn hy-repr [obj]
   (setv [f placeholder] (next
-    (genexpr (get -registry t)
-      [t (. (type obj) __mro__)]
-      (in t -registry))
+    (gfor
+      t (. (type obj) __mro__)
+      :if (in t -registry)
+      (get -registry t))
     [-base-repr None]))
 
   (global -quoting)
@@ -55,18 +56,18 @@
     ; collections.namedtuple.)
     (.format "({} {})"
       (. (type x) __name__)
-      (.join " " (genexpr (+ ":" k " " (hy-repr v)) [[k v] (zip x._fields x)])))
+      (.join " " (gfor [k v] (zip x._fields x) (+ ":" k " " (hy-repr v)))))
     ; Otherwise, print it as a regular tuple.
     (+ "(," (if x " " "") (-cat x) ")"))))
 (hy-repr-register dict :placeholder "{...}" (fn [x]
-  (setv text (.join "  " (genexpr
-    (+ (hy-repr k) " " (hy-repr v))
-    [[k v] (.items x)])))
+  (setv text (.join "  " (gfor
+    [k v] (.items x)
+    (+ (hy-repr k) " " (hy-repr v)))))
   (+ "{" text "}")))
 (hy-repr-register HyDict :placeholder "{...}" (fn [x]
-  (setv text (.join "  " (genexpr
-    (+ (hy-repr k) " " (hy-repr v))
-    [[k v] (partition x)])))
+  (setv text (.join "  " (gfor
+    [k v] (partition x)
+    (+ (hy-repr k) " " (hy-repr v)))))
   (if (% (len x) 2)
     (+= text (+ "  " (hy-repr (get x -1)))))
   (+ "{" text "}")))
@@ -162,5 +163,8 @@
   ; Call (.repr x) using the first class of x that doesn't inherit from
   ; HyObject.
   (.__repr__
-    (next (genexpr t [t (. (type x) __mro__)] (not (issubclass t HyObject))))
+    (next (gfor
+      t (. (type x) __mro__)
+      :if (not (issubclass t HyObject))
+      t))
     x))
diff --git a/hy/contrib/sequences.hy b/hy/contrib/sequences.hy
index 703da2b..e684b80 100644
--- a/hy/contrib/sequences.hy
+++ b/hy/contrib/sequences.hy
@@ -11,8 +11,8 @@
    --getitem-- (fn [self n]
                  "get nth item of sequence"
                  (if (hasattr n "start")
-                   (genexpr (get self x) [x (range n.start n.stop
-                                                   (or n.step 1))])
+                   (gfor x (range n.start n.stop (or n.step 1))
+                         (get self x))
                    (do (when (neg? n)
                          ; Call (len) to force the whole
                          ; sequence to be evaluated.
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 3f0dd39..2895bb7 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -287,7 +287,7 @@ Return series of accumulated sums (or other binary function results)."
   "Return a function applying each `fs` to args, collecting results in a list."
   (setv fs (+ (, f) fs))
   (fn [&rest args &kwargs kwargs]
-    (list-comp (f #* args #** kwargs) [f fs])))
+    (lfor f fs (f #* args #** kwargs))))
 
 (defn last [coll]
   "Return last item from `coll`."
@@ -352,8 +352,8 @@ with overlap."
   (setv
    step (or step n)
    coll-clones (tee coll n)
-   slices (genexpr (islice (get coll-clones start) start None step)
-                   [start (range n)]))
+   slices (gfor start (range n)
+                (islice (get coll-clones start) start None step)))
   (if (is fillvalue -sentinel)
     (zip #* slices)
     (zip-longest #* slices :fillvalue fillvalue)))
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 05e67e9..3c65225 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -38,9 +38,9 @@ be associated in pairs."
   `(setv ~@(+ (if other-kvs
                 [c coll]
                 [])
-              #* (genexpr [`(get ~c ~k) v]
-                          [[k v] (partition (+ (, k1 v1)
-                                               other-kvs))]))))
+              #* (gfor [k v] (partition (+ (, k1 v1)
+                                           other-kvs))
+                       [`(get ~c ~k) v]))))
 
 
 (defn _with [node args body]
@@ -206,8 +206,8 @@ the second form, the second result is inserted into the third form, and so on."
   "Like `defmacro/g!`, with automatic once-only evaluation for 'o!' params.
 
 Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
-  (setv os (list-comp s [s args] (.startswith s "o!"))
-        gs (list-comp (HySymbol (+ "g!" (cut s 2))) [s os]))
+  (setv os (lfor s args :if (.startswith s "o!") s)
+        gs (lfor s os (HySymbol (+ "g!" (cut s 2)))))
   `(defmacro/g! ~name ~args
      `(do (setv ~@(interleave ~gs ~os))
           ~@~body)))
diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy
index 1f75108..ecf42a5 100644
--- a/hy/core/shadow.hy
+++ b/hy/core/shadow.hy
@@ -98,8 +98,7 @@
 (defn comp-op [op a1 a-rest]
   "Helper for shadow comparison operators"
   (if a-rest
-    (reduce (fn [x y] (and x y))
-      (list-comp (op x y) [(, x y) (zip (+ (, a1) a-rest) a-rest)]))
+    (and #* (gfor (, x y) (zip (+ (, a1) a-rest) a-rest) (op x y)))
     True))
 (defn < [a1 &rest a-rest]
   "Shadowed `<` operator perform lt comparison on `a1` by each `a-rest`."
diff --git a/hy/extra/anaphoric.hy b/hy/extra/anaphoric.hy
index 25c187e..276f72d 100644
--- a/hy/extra/anaphoric.hy
+++ b/hy/extra/anaphoric.hy
@@ -109,18 +109,18 @@
    `%*` and `%**` name the `&rest` and `&kwargs` parameters, respectively.
 
    Nesting of `#%` forms is not recommended."
-  (setv %symbols (set-comp a
-                           [a (flatten [expr])]
-                           (and (symbol? a)
-                                (.startswith a '%))))
+  (setv %symbols (sfor a (flatten [expr])
+                       :if (and (symbol? a)
+                                (.startswith a '%))
+                       a))
   `(fn [;; generate all %i symbols up to the maximum found in expr
-        ~@(genexpr (HySymbol (+ "%" (str i)))
-                   [i (range 1 (-> (list-comp (int (cut a 1))
-                                              [a %symbols]
-                                              (.isdigit (cut a 1)))
-                                   (or (, 0))
-                                   max
-                                   inc))])
+        ~@(gfor i (range 1 (-> (lfor a %symbols
+                                     :if (.isdigit (cut a 1))
+                                     (int (cut a 1)))
+                               (or (, 0))
+                               max
+                               inc))
+                (HySymbol (+ "%" (str i))))
         ;; generate the &rest parameter only if '%* is present in expr
         ~@(if (in '%* %symbols)
               '(&rest %*))
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index 44669eb..8aca7c9 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -84,5 +84,5 @@ def test_eval():
     assert eval_str('(.strip " fooooo   ")') == 'fooooo'
     assert eval_str(
         '(if True "this is if true" "this is if false")') == "this is if true"
-    assert eval_str('(list-comp (pow num 2) [num (range 100)] (= (% num 2) 1))') == [
+    assert eval_str('(lfor num (range 100) :if (= (% num 2) 1) (pow num 2))') == [
         pow(num, 2) for num in range(100) if num % 2 == 1]
diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy
index 1f17912..9d97cbd 100644
--- a/tests/native_tests/comprehensions.hy
+++ b/tests/native_tests/comprehensions.hy
@@ -191,9 +191,9 @@
   (assert (= s "az"))
 
   (assert (= (list ((fn [] (for [x [[1] [2 3]] y x] (yield y)))))
-             (list-comp y [x [[1] [2 3]] y x])))
+             (lfor  x [[1] [2 3]]  y x  y)))
   (assert (= (list ((fn [] (for [x [[1] [2 3]] y x z (range 5)] (yield z)))))
-             (list-comp z [x [[1] [2 3]] y x z (range 5)]))))
+             (lfor  x [[1] [2 3]]  y x  z (range 5)  z))))
 
 
 (defn test-nasty-for-nesting []
diff --git a/tests/native_tests/contrib/hy_repr.hy b/tests/native_tests/contrib/hy_repr.hy
index cdea7c4..2944d85 100644
--- a/tests/native_tests/contrib/hy_repr.hy
+++ b/tests/native_tests/contrib/hy_repr.hy
@@ -144,11 +144,11 @@
 
   (setv x {1 2  3 [4 5]  6 7})
   (setv (get x 3 1) x)
-  (assert (in (hy-repr x) (list-comp
+  (assert (in (hy-repr x) (lfor
     ; The ordering of a dictionary isn't guaranteed, so we need
     ; to check for all possible orderings.
-    (+ "{" (.join "  " p) "}")
-    [p (permutations ["1 2" "3 [4 {...}]" "6 7"])]))))
+    p (permutations ["1 2" "3 [4 {...}]" "6 7"])
+    (+ "{" (.join "  " p) "}")))))
 
 (defn test-matchobject []
   (import re)
diff --git a/tests/native_tests/tag_macros.hy b/tests/native_tests/tag_macros.hy
index 5efedd1..4fb3659 100644
--- a/tests/native_tests/tag_macros.hy
+++ b/tests/native_tests/tag_macros.hy
@@ -110,7 +110,7 @@
     ((wraps func)
        (fn [&rest args &kwargs kwargs]
          (func #* (map inc args)
-               #** (dict-comp k (inc v) [[k v] (.items kwargs)])))))
+               #** (dfor [k v] (.items kwargs) [k (inc v)])))))
 
   #@(increment-arguments
      (defn foo [&rest args &kwargs kwargs]
diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
index 27d0ee4..b215375 100644
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -29,10 +29,10 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little
 (setv myset #{4 5 6})
 (setv mydict {7 8  9 900  10 15})
 
-(setv mylistcomp (list-comp x [x (range 10)] (% x 2)))
-(setv mysetcomp (set-comp x [x (range 5)] (not (% x 2))))
-(setv mydictcomp (dict-comp k (.upper k) [k "abcde"] (!= k "c")))
-(setv mygenexpr (genexpr x [x (cycle [1 2 3])] (!= x 2)))
+(setv mylistcomp (lfor x (range 10) :if (% x 2) x))
+(setv mysetcomp (sfor x (range 5) :if (not (% x 2)) x))
+(setv mydictcomp (dfor k "abcde" :if (!= k "c") [k (.upper k)]))
+(setv mygenexpr (gfor x (cycle [1 2 3]) :if (!= x 2) x))
 
 (setv attr-ref str.upper)
 (setv subscript (get "hello" 2))

From 14979edcab424d75005d93ce7d5811ba21abbd26 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 7 Jun 2018 12:20:23 -0700
Subject: [PATCH 018/223] Remove tests of the old comprehension forms

---
 tests/compilers/test_ast.py    |  8 ------
 tests/native_tests/language.hy | 46 ----------------------------------
 2 files changed, 54 deletions(-)

diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index ad41b01..bbf964a 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -554,14 +554,6 @@ def test_attribute_empty():
     cant_compile('[2].foo')
 
 
-def test_invalid_list_comprehension():
-    """Ensure that invalid list comprehensions do not break the compiler"""
-    cant_compile("(genexpr x [])")
-    cant_compile("(genexpr [x [1 2 3 4]] x)")
-    cant_compile("(list-comp None [])")
-    cant_compile("(list-comp [x [1 2 3]] x)")
-
-
 def test_bad_setv():
     """Ensure setv handles error cases"""
     cant_compile("(setv (a b) [1 2])")
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 9ce8da2..37f4d4d 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -842,52 +842,6 @@
   (assert (= x 3)))
 
 
-(defn test-list-comprehensions []
-  "NATIVE: test list comprehensions"
-  (assert (= (list-comp (* x 2) [x (range 2)]) [0 2]))
-  (assert (= (list-comp (* x 2) [x (range 4)] (% x 2)) [2 6]))
-  (assert (= (sorted (list-comp (* y 2) [(, x y) (.items {"1" 1 "2" 2})]))
-             [2 4]))
-  (assert (= (list-comp (, x y) [x (range 2) y (range 2)])
-             [(, 0 0) (, 0 1) (, 1 0) (, 1 1)]))
-  (assert (= (list-comp j [j [1 2]]) [1 2])))
-
-
-(defn test-set-comprehensions []
-  "NATIVE: test set comprehensions"
-  (assert (instance? set (set-comp x [x (range 2)])))
-  (assert (= (set-comp (* x 2) [x (range 2)]) (set [0 2])))
-  (assert (= (set-comp (* x 2) [x (range 4)] (% x 2)) (set [2 6])))
-  (assert (= (set-comp (* y 2) [(, x y) (.items {"1" 1 "2" 2})])
-             (set [2 4])))
-  (assert (= (set-comp (, x y) [x (range 2) y (range 2)])
-             (set [(, 0 0) (, 0 1) (, 1 0) (, 1 1)])))
-  (assert (= (set-comp j [j [1 2]]) (set [1 2]))))
-
-
-(defn test-dict-comprehensions []
-  "NATIVE: test dict comprehensions"
-  (assert (instance? dict (dict-comp x x [x (range 2)])))
-  (assert (= (dict-comp x (* x 2) [x (range 2)]) {1 2 0 0}))
-  (assert (= (dict-comp x (* x 2) [x (range 4)] (% x 2)) {3 6 1 2}))
-  (assert (= (dict-comp x (* y 2) [(, x y) (.items {"1" 1 "2" 2})])
-             {"2" 4 "1" 2}))
-  (assert (= (dict-comp (, x y) (+ x y) [x (range 2) y (range 2)])
-             {(, 0 0) 0 (, 1 0) 1 (, 0 1) 1 (, 1 1) 2})))
-
-
-(defn test-generator-expressions []
-  "NATIVE: test generator expressions"
-  (assert (not (instance? list (genexpr x [x (range 2)]))))
-  (assert (= (list (genexpr (* x 2) [x (range 2)])) [0 2]))
-  (assert (= (list (genexpr (* x 2) [x (range 4)] (% x 2))) [2 6]))
-  (assert (= (list (sorted (genexpr (* y 2) [(, x y) (.items {"1" 1 "2" 2})])))
-             [2 4]))
-  (assert (= (list (genexpr (, x y) [x (range 2) y (range 2)]))
-             [(, 0 0) (, 0 1) (, 1 0) (, 1 1)]))
-  (assert (= (list (genexpr j [j [1 2]])) [1 2])))
-
-
 (defn test-defn-order []
   "NATIVE: test defn evaluation order"
   (setv acc [])

From 76b80bad813a655ed1715d4f1203413257d757f2 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 7 Jun 2018 12:43:04 -0700
Subject: [PATCH 019/223] Remove support for the old comprehension forms

---
 hy/compiler.py | 37 +------------------------------------
 1 file changed, 1 insertion(+), 36 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index d6bc53d..2d065c2 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1020,41 +1020,6 @@ class HyASTCompiler(object):
 
         return gen_res + cond, gen
 
-    @special(["list-comp", "set-comp", "genexpr"], [FORM, FORM, maybe(FORM)])
-    def compile_comprehension(self, expr, form, expression, gen, cond):
-        # (list-comp expr [target iter] cond?)
-
-        if not isinstance(gen, HyList):
-            raise HyTypeError(gen, "Generator expression must be a list.")
-
-        gen_res, gen = self._compile_generator_iterables(
-            [gen] + ([] if cond is None else [cond]))
-
-        if len(gen) == 0:
-            raise HyTypeError(expr, "Generator expression cannot be empty.")
-
-        ret = self.compile(expression)
-        node_class = (
-            asty.ListComp if form == "list-comp" else
-            asty.SetComp if form == "set-comp" else
-            asty.GeneratorExp)
-        return ret + gen_res + node_class(
-            expr, elt=ret.force_expr, generators=gen)
-
-    @special("dict-comp", [FORM, FORM, FORM, maybe(FORM)])
-    def compile_dict_comprehension(self, expr, root, key, value, gen, cond):
-        key = self.compile(key)
-        value = self.compile(value)
-
-        gen_res, gen = self._compile_generator_iterables(
-            [gen] + ([] if cond is None else [cond]))
-
-        return key + value + gen_res + asty.DictComp(
-            expr,
-            key=key.force_expr,
-            value=value.force_expr,
-            generators=gen)
-
     _loopers = many(
         tag('setv', sym(":setv") + FORM + FORM) |
         tag('if', sym(":if") + FORM) |
@@ -1065,7 +1030,7 @@ class HyASTCompiler(object):
         many(notpexpr("else")) + maybe(dolike("else"))])
     @special(["lfor", "sfor", "gfor"], [_loopers, FORM])
     @special(["dfor"], [_loopers, brackets(FORM, FORM)])
-    def compile_new_comp(self, expr, root, parts, final):
+    def compile_comprehension(self, expr, root, parts, final):
         root = unmangle(ast_str(root))
         node_class = {
             "for":  asty.For,

From da754c0e5d4b7e6a2f34b0345c6c694b27319cea Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 12 Jun 2018 11:13:06 -0700
Subject: [PATCH 020/223] Update NEWS and docs for the new comprehensions

---
 NEWS.rst                |   6 +
 docs/extra/reserved.rst |   2 +-
 docs/language/api.rst   | 255 +++++++++++++++++++++-------------------
 docs/tutorial.rst       |  17 ++-
 4 files changed, 149 insertions(+), 131 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 35155f2..5ca0d90 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -14,6 +14,7 @@ Removals
 * Macros `ap-pipe` and `ap-compose` have been removed.
   Anaphoric macros do not work well with point-free style programming,
   in which case both threading macros and `comp` are more adequate.
+* `for/a` has been removed. Use `(for [:async ...] ...)` instead.
 
 Other Breaking Changes
 ------------------------------
@@ -30,6 +31,10 @@ Other Breaking Changes
 * Non-shadow unary `=`, `is`, `<`, etc. now evaluate their argument
   instead of ignoring it. This change increases consistency a bit
   and makes accidental unary uses easier to notice.
+* `list-comp`, `set-comp`, `dict-comp`, and `genexpr` have been replaced
+  by `lfor`, `sfor`, `dfor`, and `gfor`, respectively, which use a new
+  syntax and have additional features. All Python comprehensions can now
+  be written in Hy.
 * `hy-repr` uses registered functions instead of methods
 * `HyKeyword` no longer inherits from the string type and has been
   made into its own object type.
@@ -47,6 +52,7 @@ New Features
   keyword arguments
 * Added a command-line option `-E` per CPython
 * `while` and `for` are allowed to have empty bodies
+* `for` supports the various new clause types offered by `lfor`
 * Added a new module ``hy.model_patterns``
 
 Bug Fixes
diff --git a/docs/extra/reserved.rst b/docs/extra/reserved.rst
index 26f853f..9ef4639 100644
--- a/docs/extra/reserved.rst
+++ b/docs/extra/reserved.rst
@@ -10,7 +10,7 @@ Usage: ``(names)``
 This function can be used to get a list (actually, a ``frozenset``) of the
 names of Hy's built-in functions, macros, and special forms. The output
 also includes all Python reserved words. All names are in unmangled form
-(e.g., ``list-comp`` rather than ``list_comp``).
+(e.g., ``not-in`` rather than ``not_in``).
 
 .. code-block:: hy
 
diff --git a/docs/language/api.rst b/docs/language/api.rst
index 73066ee..68c9280 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -321,48 +321,17 @@ is only called on every other value in the list.
       (side-effect2 x))
 
 
-dict-comp
----------
-
-``dict-comp`` is used to create dictionaries. It takes three or four parameters.
-The first two parameters are for controlling the return value (key-value pair)
-while the third is used to select items from a sequence. The fourth and optional
-parameter can be used to filter out some of the items in the sequence based on a
-conditional expression.
-
-.. code-block:: hy
-
-    => (dict-comp x (* x 2) [x (range 10)] (odd? x))
-    {1: 2, 3: 6, 9: 18, 5: 10, 7: 14}
-
-
 do
 ----------
 
-``do`` is used to evaluate each of its arguments and return the
-last one. Return values from every other than the last argument are discarded.
-It can be used in ``list-comp`` to perform more complex logic as shown in one
-of the following examples.
+``do`` (called ``progn`` in some Lisps) takes any number of forms,
+evaluates them, and returns the value of the last one, or ``None`` if no
+forms were provided.
 
-Some example usage:
+::
 
-.. code-block:: clj
-
-    => (if True
-    ...  (do (print "Side effects rock!")
-    ...      (print "Yeah, really!")))
-    Side effects rock!
-    Yeah, really!
-
-    ;; assuming that (side-effect) is a function that we want to call for each
-    ;; and every value in the list, but whose return value we do not care about
-    => (list-comp (do (side-effect x)
-    ...               (if (< x 5) (* 2 x)
-    ...                   (* 4 x)))
-    ...           (x (range 10)))
-    [0, 2, 4, 6, 8, 20, 24, 28, 32, 36]
-
-``do`` can accept any number of arguments, from 1 to n.
+    => (+ 1 (do (setv x (+ 1 1)) x))
+    3
 
 
 doc / #doc
@@ -400,6 +369,20 @@ Gets help for macros or tag macros, respectively.
     Gets help for a tag macro function available in this module.
 
 
+dfor
+----
+
+``dfor`` creates a :ref:`dictionary comprehension <py:dict>`. Its syntax
+is the same as that of `lfor`_ except that the final value form must be
+a literal list of two elements, the first of which becomes each key and
+the second of which becomes each value.
+
+.. code-block:: hy
+
+    => (dfor x (range 5) [x (* x 10)])
+    {0: 0, 1: 10, 2: 20, 3: 30, 4: 40}
+
+
 setv
 ----
 
@@ -524,8 +507,8 @@ Parameters may have the following keywords in front of them:
     .. code-block:: clj
 
         => (defn zig-zag-sum [&rest numbers]
-             (setv odd-numbers (list-comp x [x numbers] (odd? x))
-                   even-numbers (list-comp x [x numbers] (even? x)))
+             (setv odd-numbers (lfor x numbers :if (odd? x) x)
+                   even-numbers (lfor x numbers :if (even? x) x))
              (- (sum odd-numbers) (sum even-numbers)))
 
         => (zig-zag-sum)
@@ -850,24 +833,39 @@ raising an exception.
 for
 ---
 
-``for`` is used to call a function for each element in a list or vector.
-The results of each call are discarded and the ``for`` expression returns
-``None`` instead. The example code iterates over *collection* and for each
-*element* in *collection* calls the ``side-effect`` function with *element*
-as its argument:
+``for`` is used to evaluate some forms for each element in an iterable
+object, such as a list. The return values of the forms are discarded and
+the ``for`` form returns ``None``.
 
-.. code-block:: clj
+::
 
-    ;; assuming that (side-effect) is a function that takes a single parameter
-    (for [element collection] (side-effect element))
+    => (for [x [1 2 3]]
+    ...  (print "iterating")
+    ...  (print x))
+    iterating
+    1
+    iterating
+    2
+    iterating
+    3
 
-    ;; for can have an optional else block
-    (for [element collection] (side-effect element)
-         (else (side-effect-2)))
+In its square-bracketed first argument, ``for`` allows the same types of
+clauses as lfor_.
 
-The optional ``else`` block is only executed if the ``for`` loop terminates
-normally. If the execution is halted with ``break``, the ``else`` block does
-not execute.
+::
+
+   => (for [x [1 2 3]  :if (!= x 2)  y [7 8]]
+   ...  (print x y))
+   1 7
+   1 8
+   3 7
+   3 8
+
+Furthermore, the last argument of ``for`` can be an ``(else …)`` form.
+This form is executed after the last iteration of the ``for``\'s
+outermost iteration clause, but only if that outermost loop terminates
+normally. If it's jumped out of with e.g. ``break``, the ``else`` is
+ignored.
 
 .. code-block:: clj
 
@@ -888,43 +886,6 @@ not execute.
     loop finished
 
 
-for/a
------
-
-``for/a`` behaves like ``for`` but is used to call a function for each
-element generated by an asynchronous generator expression. The results
-of each call are discarded and the ``for/a`` expression returns
-``None`` instead.
-
-.. code-block:: clj
-
-    ;; assuming that (side-effect) is a function that takes a single parameter
-    (for/a [element (agen)] (side-effect element))
-
-    ;; for/a can have an optional else block
-    (for/a [element (agen)] (side-effect element)
-         (else (side-effect-2)))
-
-
-genexpr
--------
-
-``genexpr`` is used to create generator expressions. It takes two or three
-parameters. The first parameter is the expression controlling the return value,
-while the second is used to select items from a list. The third and optional
-parameter can be used to filter out some of the items in the list based on a
-conditional expression. ``genexpr`` is similar to ``list-comp``, except it
-returns an iterable that evaluates values one by one instead of evaluating them
-immediately.
-
-.. code-block:: hy
-
-    => (setv collection (range 10))
-    => (setv filtered (genexpr x [x collection] (even? x)))
-    => (list filtered)
-    [0, 2, 4, 6, 8]
-
-
 .. _gensym:
 
 gensym
@@ -977,6 +938,24 @@ successive elements in a nested structure. Example usage:
           index that is out of bounds.
 
 
+gfor
+----
+
+``gfor`` creates a :ref:`generator expression <py:genexpr>`. Its syntax
+is the same as that of `lfor`_. The difference is that ``gfor`` returns
+an iterator, which evaluates and yields values one at a time.
+
+::
+
+    => (setv accum [])
+    => (list (take-while
+    ...  (fn [x] (< x 5))
+    ...  (gfor x (count) :do (.append accum x) x)))
+    [0, 1, 2, 3, 4]
+    => accum
+    [0, 1, 2, 3, 4, 5]
+
+
 global
 ------
 
@@ -1190,27 +1169,69 @@ last
     6
 
 
-list-comp
----------
+lfor
+----
 
-``list-comp`` performs list comprehensions. It takes two or three parameters.
-The first parameter is the expression controlling the return value, while
-the second is used to select items from a list. The third and optional
-parameter can be used to filter out some of the items in the list based on a
-conditional expression. Some examples:
+The comprehension forms ``lfor``, `sfor`_, `dfor`_, `gfor`_, and `for`_
+are used to produce various kinds of loops, including Python-style
+:ref:`comprehensions <py:comprehensions>`. ``lfor`` in particular
+creates a list comprehension. A simple use of ``lfor`` is::
 
-.. code-block:: clj
-
-    => (setv collection (range 10))
-    => (list-comp x [x collection])
-    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-
-    => (list-comp (* x 2) [x collection])
-    [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
-
-    => (list-comp (* x 2) [x collection] (< x 5))
+    => (lfor x (range 5) (* 2 x))
     [0, 2, 4, 6, 8]
 
+``x`` is the name of a new variable, which is bound to each element of
+``(range 5)``. Each such element in turn is used to evaluate the value
+form ``(* 2 x)``, and the results are accumulated into a list.
+
+Here's a more complex example::
+
+    => (lfor
+    ...  x (range 3)
+    ...  y (range 3)
+    ...  :if (!= x y)
+    ...  :setv total (+ x y)
+    ...  [x y total])
+    [[0, 1, 1], [0, 2, 2], [1, 0, 1], [1, 2, 3], [2, 0, 2], [2, 1, 3]]
+
+When there are several iteration clauses (here, the pairs of forms ``x
+(range 3)`` and ``y (range 3)``), the result works like a nested loop or
+Cartesian product: all combinations are considered in lexicographic
+order.
+
+The general form of ``lfor`` is::
+
+    (lfor CLAUSES VALUE)
+
+where the ``VALUE`` is an arbitrary form that is evaluated to produce
+each element of the result list, and ``CLAUSES`` is any number of
+clauses. There are several types of clauses:
+
+- Iteration clauses, which look like ``LVALUE ITERABLE``. The ``LVALUE``
+  is usually just a symbol, but could be something more complicated,
+  like ``[x y]``.
+- ``:async LVALUE ITERABLE``, which is an
+  :ref:`asynchronous <py:async for>` form of iteration clause.
+- ``:do FORM``, which simply evaluates the ``FORM``. If you use
+  ``(continue)`` or ``(break)`` here, they will apply to the innermost
+  iteration clause before the ``:do``.
+- ``:setv LVALUE RVALUE``, which is equivalent to ``:do (setv LVALUE
+  RVALUE)``.
+- ``:if CONDITION``, which is equivalent to ``:do (unless CONDITION
+  (continue))``.
+
+For ``lfor``, ``sfor``, ``gfor``, and ``dfor``, variables are scoped as
+if the comprehension form were its own function, so variables defined by
+an iteration clause or ``:setv`` are not visible outside the form. In
+fact, these forms are implemented as generator functions whenever they
+contain Python statements, with the attendant consequences for calling
+``return``. By contrast, ``for`` shares the caller's scope.
+
+.. note:: An exception to the above scoping rules occurs on Python 2 for
+   ``lfor`` specifically (and not ``sfor``, ``gfor``, or ``dfor``) when
+   Hy can implement the ``lfor`` as a Python list comprehension. Then,
+   variables will leak to the surrounding scope.
+
 
 nonlocal
 --------
@@ -1465,20 +1486,12 @@ the end of a function, put ``None`` there yourself.
     => (print (f 4))
     None
 
-set-comp
---------
 
-``set-comp`` is used to create sets. It takes two or three parameters.
-The first parameter is for controlling the return value, while the second is
-used to select items from a sequence. The third and optional parameter can be
-used to filter out some of the items in the sequence based on a conditional
-expression.
+sfor
+----
 
-.. code-block:: hy
-
-    => (setv data [1 2 3 4 5 2 3 4 5 3 4 5])
-    => (set-comp x [x data] (odd? x))
-    {1, 3, 5}
+``sfor`` creates a set comprehension. ``(sfor CLAUSES VALUE)`` is
+equivalent to ``(set (lfor CLAUSES VALUE))``. See `lfor`_.
 
 
 cut
@@ -1944,13 +1957,13 @@ infinite series without consuming infinite amount of memory.
     => (multiply (range 5) (range 5))
     <generator object multiply at 0x978d8ec>
 
-    => (list-comp value [value (multiply (range 10) (range 10))])
+    => (list (multiply (range 10) (range 10)))
     [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
 
     => (import random)
     => (defn random-numbers [low high]
     ...  (while True (yield (.randint random low high))))
-    => (list-comp x [x (take 15 (random-numbers 1 50))])
+    => (list (take 15 (random-numbers 1 50)))
     [7, 41, 6, 22, 32, 17, 5, 38, 18, 38, 17, 14, 23, 23, 19]
 
 
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index b85bf3a..70522e6 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -378,21 +378,20 @@ In Hy, you could do these like:
 .. code-block:: clj
 
   (setv odds-squared
-    (list-comp
-      (pow num 2)
-      (num (range 100))
-      (= (% num 2) 1)))
-
+    (lfor
+      num (range 100)
+      :if (= (% num 2) 1)
+      (pow num 2)))
 
 .. code-block:: clj
 
   ; And, an example stolen shamelessly from a Clojure page:
   ; Let's list all the blocks of a Chessboard:
 
-  (list-comp
-    (, x y)
-    (x (range 8)
-     y "ABCDEFGH"))
+  (lfor
+    x (range 8)
+    y "ABCDEFGH"
+    (, x y))
 
   ; [(0, 'A'), (0, 'B'), (0, 'C'), (0, 'D'), (0, 'E'), (0, 'F'), (0, 'G'), (0, 'H'),
   ;  (1, 'A'), (1, 'B'), (1, 'C'), (1, 'D'), (1, 'E'), (1, 'F'), (1, 'G'), (1, 'H'),

From edbe8e3b7f589523b79cc81b61be687c8e1edcf3 Mon Sep 17 00:00:00 2001
From: Oskar Kvist <oskar.kvist@gmail.com>
Date: Mon, 11 Jun 2018 17:37:31 +0200
Subject: [PATCH 021/223] Make defmacro! work with optional args

---
 NEWS.rst                            | 1 +
 hy/core/macros.hy                   | 7 ++++++-
 tests/native_tests/native_macros.hy | 9 ++++++++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 5ca0d90..159fdfd 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -54,6 +54,7 @@ New Features
 * `while` and `for` are allowed to have empty bodies
 * `for` supports the various new clause types offered by `lfor`
 * Added a new module ``hy.model_patterns``
+* `defmacro!` now allows optional args
 
 Bug Fixes
 ------------------------------
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 3c65225..c607c74 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -206,7 +206,12 @@ the second form, the second result is inserted into the third form, and so on."
   "Like `defmacro/g!`, with automatic once-only evaluation for 'o!' params.
 
 Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
-  (setv os (lfor s args :if (.startswith s "o!") s)
+  (defn extract-o!-sym [arg]
+    (cond [(and (symbol? arg) (.startswith arg "o!"))
+           arg]
+          [(and (instance? list arg) (.startswith (first arg) "o!"))
+           (first arg)]))
+  (setv os (list (filter identity (map extract-o!-sym args)))
         gs (lfor s os (HySymbol (+ "g!" (cut s 2)))))
   `(defmacro/g! ~name ~args
      `(do (setv ~@(interleave ~gs ~os))
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 79ae560..e58c911 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -257,7 +257,14 @@
   ;; test that o! is evaluated once only
   (setv foo 40)
   (foo! (+= foo 1))
-  (assert (= 41 foo)))
+  (assert (= 41 foo))
+  ;; test &optional args
+  (defmacro! bar! [o!a &optional [o!b 1]] `(do ~g!a ~g!a ~g!b ~g!b))
+  ;; test that o!s are evaluated once only
+  (bar! (+= foo 1) (+= foo 1))
+  (assert (= 43 foo))
+  ;; test that the optional arg works
+  (assert (= (bar! 2) 1)))
 
 
 (defn test-if-not []

From 97c15c1bb9d3c99ab3fb0cbbb236d5e279afe4f7 Mon Sep 17 00:00:00 2001
From: Oskar Kvist <oskar.kvist@gmail.com>
Date: Mon, 11 Jun 2018 17:49:11 +0200
Subject: [PATCH 022/223] Add Oskar Kvist to AUTHORS

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 6f17386..5ffa1df 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -88,3 +88,4 @@
 * Yoan Tournade <yoan@ytotech.com>
 * Simon Gomizelj <simon@vodik.xyz>
 * Yigong Wang <wang@yigo.ng>
+* Oskar Kvist <oskar.kvist@gmail.com>

From 7abd8ffc2a6613c540c20877735e28b5c68370d3 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 13 Jun 2018 09:04:52 -0700
Subject: [PATCH 023/223] Make importing a dotted name a syntax error, per
 Python

---
 NEWS.rst                    | 1 +
 hy/compiler.py              | 6 ++++--
 tests/compilers/test_ast.py | 7 +++++++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 159fdfd..bd46cb9 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -67,6 +67,7 @@ Bug Fixes
 * `NaN` can no longer create an infinite loop during macro-expansion
 * Fixed a bug that caused `try` to drop expressions
 * Fixed a bug where the compiler didn't properly compile `unquote-splice`
+* Trying to import a dotted name is now a syntax error, as in Python
 
 Misc. Improvements
 ----------------------------
diff --git a/hy/compiler.py b/hy/compiler.py
index 2d065c2..04d195d 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1176,10 +1176,12 @@ class HyASTCompiler(object):
 
         return operand
 
+    _symn = some(lambda x: isinstance(x, HySymbol) and "." not in x)
+
     @special(["import", "require"], [many(
         SYM |
-        brackets(SYM, sym(":as"), SYM) |
-        brackets(SYM, brackets(many(SYM + maybe(sym(":as") + SYM)))))])
+        brackets(SYM, sym(":as"), _symn) |
+        brackets(SYM, brackets(many(_symn + maybe(sym(":as") + _symn)))))])
     def compile_import_or_require(self, expr, root, entries):
         """
         TODO for `require`: keep track of what we've imported in this run and
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index bbf964a..5fbc727 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -283,6 +283,13 @@ def test_ast_require():
     cant_compile("(require [tests.resources.tlib [* *]])")
 
 
+def test_ast_import_require_dotted():
+    """As in Python, it should be a compile-type error to attempt to
+import a dotted name."""
+    cant_compile("(import [spam [foo.bar]])")
+    cant_compile("(require [spam [foo.bar]])")
+
+
 def test_ast_no_pointless_imports():
     def contains_import_from(code):
         return any([isinstance(node, ast.ImportFrom)

From 9a8886a452698523c4184b612f89fc9a6785af6f Mon Sep 17 00:00:00 2001
From: gilch <gilch@users.noreply.github.com>
Date: Wed, 27 Jun 2018 23:38:06 -0600
Subject: [PATCH 024/223] Proper special indent in let tests.

---
 tests/native_tests/contrib/walk.hy | 286 ++++++++++++++---------------
 1 file changed, 143 insertions(+), 143 deletions(-)

diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index 7facd59..2156c3e 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -64,16 +64,16 @@
         b "b")
   (let [a "x"
         b "y"]
-       (assert (= (+ a b)
-                  "xy"))
-       (let [a "z"]
-            (assert (= (+ a b)
-                       "zy")))
-       ;; let-shadowed variable doesn't get clobbered.
-       (assert (= (+ a b)
-                  "xy")))
+    (assert (= (+ a b)
+               "xy"))
+    (let [a "z"]
+      (assert (= (+ a b)
+                 "zy")))
+    ;; let-shadowed variable doesn't get clobbered.
+    (assert (= (+ a b)
+               "xy")))
   (let [q "q"]
-       (assert (= q "q")))
+    (assert (= q "q")))
   (assert (= a "a"))
   (assert (= b "b"))
   (assert (in "a" (.keys (vars))))
@@ -85,86 +85,86 @@
   (let [a "a"
         b "b"
         ab (+ a b)]
-       (assert (= ab "ab"))
-       (let [c "c"
-             abc (+ ab c)]
-            (assert (= abc "abc")))))
+    (assert (= ab "ab"))
+    (let [c "c"
+          abc (+ ab c)]
+      (assert (= abc "abc")))))
 
 (defn test-let-early []
   (setv a "a")
   (let [q (+ a "x")
         a 2  ; should not affect q
         b 3]
-       (assert (= q "ax"))
-       (let [q (* a b)
-             a (+ a b)
-             b (* a b)]
-            (assert (= q 6))
-            (assert (= a 5))
-            (assert (= b 15))))
+    (assert (= q "ax"))
+    (let [q (* a b)
+          a (+ a b)
+          b (* a b)]
+      (assert (= q 6))
+      (assert (= a 5))
+      (assert (= b 15))))
   (assert (= a "a")))
 
 (defn test-let-special []
   ;; special forms in function position still work as normal
   (let [, 1]
-       (assert (= (, , ,)
-                  (, 1 1)))))
+    (assert (= (, , ,)
+               (, 1 1)))))
 
 (defn test-let-quasiquote []
   (setv a-symbol 'a)
   (let [a "x"]
-       (assert (= a "x"))
-       (assert (= 'a a-symbol))
-       (assert (= `a a-symbol))
-       (assert (= `(foo ~a)
-                  '(foo "x")))
-       (assert (= `(foo `(bar a ~a ~~a))
-                  '(foo `(bar a ~a ~"x"))))
-       (assert (= `(foo ~@[a])
-                  '(foo "x")))
-       (assert (= `(foo `(bar [a] ~@[a] ~@~[a 'a `a] ~~@[a]))
-                  '(foo `(bar [a] ~@[a] ~@["x" a a] ~"x"))))))
+    (assert (= a "x"))
+    (assert (= 'a a-symbol))
+    (assert (= `a a-symbol))
+    (assert (= `(foo ~a)
+               '(foo "x")))
+    (assert (= `(foo `(bar a ~a ~~a))
+               '(foo `(bar a ~a ~"x"))))
+    (assert (= `(foo ~@[a])
+               '(foo "x")))
+    (assert (= `(foo `(bar [a] ~@[a] ~@~[a 'a `a] ~~@[a]))
+               '(foo `(bar [a] ~@[a] ~@["x" a a] ~"x"))))))
 
 (defn test-let-except []
   (let [foo 42
         bar 33]
-       (assert (= foo 42))
-       (try
-         (do
-           1/0
-           (assert False))
-         (except [foo Exception]
-           ;; let bindings should work in except block
-           (assert (= bar 33))
-           ;; but exception bindings can shadow let bindings
-           (assert (instance? Exception foo))))
-       ;; let binding did not get clobbered.
-       (assert (= foo 42))))
+    (assert (= foo 42))
+    (try
+      (do
+        1/0
+        (assert False))
+      (except [foo Exception]
+        ;; let bindings should work in except block
+        (assert (= bar 33))
+        ;; but exception bindings can shadow let bindings
+        (assert (instance? Exception foo))))
+    ;; let binding did not get clobbered.
+    (assert (= foo 42))))
 
 (defn test-let-mutation []
   (setv foo 42)
   (setv error False)
   (let [foo 12
         bar 13]
-       (assert (= foo 12))
-       (setv foo 14)
-       (assert (= foo 14))
-       (del foo)
-       ;; deleting a let binding should not affect others
-       (assert (= bar 13))
-       (try
-         ;; foo=42 is still shadowed, but the let binding was deleted.
-         (do
-           foo
-           (assert False))
-         (except [le LookupError]
-           (setv error le)))
-       (setv foo 16)
-       (assert (= foo 16))
-       (setv [foo bar baz] [1 2 3])
-       (assert (= foo 1))
-       (assert (= bar 2))
-       (assert (= baz 3)))
+    (assert (= foo 12))
+    (setv foo 14)
+    (assert (= foo 14))
+    (del foo)
+    ;; deleting a let binding should not affect others
+    (assert (= bar 13))
+    (try
+      ;; foo=42 is still shadowed, but the let binding was deleted.
+      (do
+        foo
+        (assert False))
+      (except [le LookupError]
+        (setv error le)))
+    (setv foo 16)
+    (assert (= foo 16))
+    (setv [foo bar baz] [1 2 3])
+    (assert (= foo 1))
+    (assert (= bar 2))
+    (assert (= baz 3)))
   (assert error)
   (assert (= foo 42))
   (assert (= baz 3)))
@@ -172,40 +172,40 @@
 (defn test-let-break []
   (for [x (range 3)]
     (let [done (odd? x)]
-         (if done (break))))
+      (if done (break))))
   (assert (= x 1)))
 
 (defn test-let-continue []
   (let [foo []]
-       (for [x (range 10)]
-         (let [odd (odd? x)]
-              (if odd (continue))
-              (.append foo x)))
-       (assert (= foo [0 2 4 6 8]))))
+    (for [x (range 10)]
+      (let [odd (odd? x)]
+        (if odd (continue))
+        (.append foo x)))
+    (assert (= foo [0 2 4 6 8]))))
 
 (defn test-let-yield []
   (defn grind []
     (yield 0)
     (let [a 1
           b 2]
-         (yield a)
-         (yield b)))
+      (yield a)
+      (yield b)))
   (assert (= (tuple (grind))
              (, 0 1 2))))
 
 (defn test-let-return []
   (defn get-answer []
     (let [answer 42]
-         (return answer)))
+      (return answer)))
   (assert (= (get-answer)
              42)))
 
 (defn test-let-import []
   (let [types 6]
-       ;; imports don't fail, even if using a let-bound name
-       (import types)
-       ;; let-bound name is not affected
-       (assert (= types 6)))
+    ;; imports don't fail, even if using a let-bound name
+    (import types)
+    ;; let-bound name is not affected
+    (assert (= types 6)))
   ;; import happened in Python scope.
   (assert (in "types" (vars)))
   (assert (instance? types.ModuleType types)))
@@ -213,13 +213,13 @@
 (defn test-let-defclass []
   (let [Foo 42
         quux object]
-       ;; the name of the class is just a symbol, even if it's a let binding
-       (defclass Foo [quux]  ; let bindings apply in inheritance list
-         ;; let bindings apply inside class body
-         (setv x Foo)
-         ;; quux is not local
-         (setv quux "quux"))
-       (assert (= quux "quux")))
+    ;; the name of the class is just a symbol, even if it's a let binding
+    (defclass Foo [quux]  ; let bindings apply in inheritance list
+      ;; let bindings apply inside class body
+      (setv x Foo)
+      ;; quux is not local
+      (setv quux "quux"))
+    (assert (= quux "quux")))
   ;; defclass always creates a python-scoped variable, even if it's a let binding name
   (assert (= Foo.x 42)))
 
@@ -229,82 +229,82 @@
   (let [a 1
         b []
         bar (fn [])]
-       (setv bar.a 13)
-       (assert (= bar.a 13))
-       (setv (. bar a) 14)
-       (assert (= bar.a 14))
-       (assert (= a 1))
-       (assert (= b []))
-       ;; method syntax not affected
-       (.append b 2)
-       (assert (= b [2]))
-       ;; attrs access is not affected
-       (assert (= foo.a 42))
-       (assert (= (. foo a)
-                  42))
-       ;; but indexing is
-       (assert (= (. [1 2 3]
-                     [a])
-                  2))))
+    (setv bar.a 13)
+    (assert (= bar.a 13))
+    (setv (. bar a) 14)
+    (assert (= bar.a 14))
+    (assert (= a 1))
+    (assert (= b []))
+    ;; method syntax not affected
+    (.append b 2)
+    (assert (= b [2]))
+    ;; attrs access is not affected
+    (assert (= foo.a 42))
+    (assert (= (. foo a)
+               42))
+    ;; but indexing is
+    (assert (= (. [1 2 3]
+                  [a])
+               2))))
 
 (defn test-let-positional []
   (let [a 0
         b 1
         c 2]
-       (defn foo [a b]
-         (, a b c))
-       (assert (= (foo 100 200)
-                  (, 100 200 2)))
-       (setv c 300)
-       (assert (= (foo 1000 2000)
-                  (, 1000 2000 300)))
-       (assert (= a 0))
-       (assert (= b 1))
-       (assert (= c 300))))
+    (defn foo [a b]
+      (, a b c))
+    (assert (= (foo 100 200)
+               (, 100 200 2)))
+    (setv c 300)
+    (assert (= (foo 1000 2000)
+               (, 1000 2000 300)))
+    (assert (= a 0))
+    (assert (= b 1))
+    (assert (= c 300))))
 
 (defn test-let-rest []
   (let [xs 6
         a 88
         c 64
         &rest 12]
-       (defn foo [a b &rest xs]
-         (-= a 1)
-         (setv xs (list xs))
-         (.append xs 42)
-         (, &rest a b c xs))
-       (assert (= xs 6))
-       (assert (= a 88))
-       (assert (= (foo 1 2 3 4)
-                  (, 12 0 2 64 [3 4 42])))
-       (assert (= xs 6))
-       (assert (= c 64))
-       (assert (= a 88))))
+    (defn foo [a b &rest xs]
+      (-= a 1)
+      (setv xs (list xs))
+      (.append xs 42)
+      (, &rest a b c xs))
+    (assert (= xs 6))
+    (assert (= a 88))
+    (assert (= (foo 1 2 3 4)
+               (, 12 0 2 64 [3 4 42])))
+    (assert (= xs 6))
+    (assert (= c 64))
+    (assert (= a 88))))
 
 (defn test-let-kwargs []
   (let [kws 6
         &kwargs 13]
-       (defn foo [&kwargs kws]
-         (, &kwargs kws))
-       (assert (= kws 6))
-       (assert (= (foo :a 1)
-                  (, 13 {"a" 1})))))
+    (defn foo [&kwargs kws]
+      (, &kwargs kws))
+    (assert (= kws 6))
+    (assert (= (foo :a 1)
+               (, 13 {"a" 1})))))
 
 (defn test-let-optional []
   (let [a 1
         b 6
         d 2]
-       (defn foo [&optional [a a] b [c d]]
-         (, a b c))
-       (assert (= (foo)
-                  (, 1 None 2)))
-       (assert (= (foo 10 20 30)
-                  (, 10 20 30)))))
+    (defn foo [&optional [a a] b [c d]]
+      (, a b c))
+    (assert (= (foo)
+               (, 1 None 2)))
+    (assert (= (foo 10 20 30)
+               (, 10 20 30)))))
 
 (defn test-let-closure []
   (let [count 0]
-       (defn +count [&optional [x 1]]
-         (+= count x)
-         count))
+    (defn +count [&optional [x 1]]
+      (+= count x)
+      count))
   ;; let bindings can still exist outside of a let body
   (assert (= 1 (+count)))
   (assert (= 2 (+count)))
@@ -323,9 +323,9 @@
   (let [a 1
         b (triple a)
         c (ap-triple)]
-       (assert (= (triple a)
-                  3))
-       (assert (= (ap-triple)
-                  3))
-       (assert (= b 3))
-       (assert (= c 3))))
+    (assert (= (triple a)
+               3))
+    (assert (= (ap-triple)
+               3))
+    (assert (= b 3))
+    (assert (= c 3))))

From 4b0e318997de2adb50a599c93a7b7cefa6d0b1f1 Mon Sep 17 00:00:00 2001
From: gilch <gilch@users.noreply.github.com>
Date: Wed, 27 Jun 2018 23:39:44 -0600
Subject: [PATCH 025/223] Remove outdated comment in walk.

---
 hy/contrib/walk.hy | 57 ++++------------------------------------------
 1 file changed, 4 insertions(+), 53 deletions(-)

diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index 8e9b3c3..f7d5626 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -313,63 +313,14 @@ as can nested let forms.
             (macro-error k "bind targets must be symbols")
             (if (in '. k)
                 (macro-error k "binding target may not contain a dot")))
-    (.append values (symbolexpand (macroexpand-all v &name) expander))
+    (.append values (symbolexpand (macroexpand-all v &name)
+                                  expander))
     (assoc replacements k `(get ~g!let ~(name k))))
   `(do
      (setv ~g!let {}
            ~@(interleave (.values replacements) values))
-     ~@(symbolexpand (macroexpand-all body &name) expander)))
+     ~@(symbolexpand (macroexpand-all body &name)
+                     expander)))
 
 ;; (defmacro macrolet [])
 
-#_[special cases for let
-   ;; Symbols containing a dot should be converted to this form.
-   ;; attrs should not get expanded,
-   ;; but [] lookups should.
-   '.',
-
-   ;;; can shadow let bindings with Python locals
-   ;; protect its bindings for the lexical scope of its body.
-   'fn',
-   'fn*',
-   ;; protect as bindings for the lexical scope of its body
-   'except',
-
-   ;;; changes scope of named variables
-   ;; protect the variables they name for the lexical scope of their container
-   'global',
-   'nonlocal',
-   ;; should we provide a protect form?
-   ;; it's an anaphor only valid in a `let` body.
-   ;; this would make the named variables python-scoped in its body
-   ;; expands to a do
-   'protect',
-
-   ;;; quoted variables must not be expanded.
-   ;; but unprotected, unquoted variables must be.
-   'quasiquote',
-   'quote',
-   'unquote',
-   'unquote-splice',
-
-   ;;;; deferred
-
-   ;; should really only exist at toplevel. Ignore until someone complains?
-   ;; raise an error? treat like fn?
-   ;; should probably be implemented as macros in terms of fn/setv anyway.
-   'defmacro',
-   'deftag',
-
-   ;;; create Python-scoped variables. It's probably hard to avoid this.
-   ;; Best just doc this behavior for now.
-   ;; we can't avoid clobbering enclosing python scope, unless we use a gensym,
-   ;; but that corrupts '__name__'.
-   ;; It could be set later, but that could mess up metaclasses!
-   ;; Should the class name update let variables too?
-   'defclass',
-   ;; should this update let variables?
-   ;; it could be done with gensym/setv.
-   'import',
-
-   ;; I don't understand these. Ignore until someone complains?
-   'eval_and_compile', 'eval_when_compile', 'require',]

From 8c79015b40ee2dc592e964abcdb21dfa414828c8 Mon Sep 17 00:00:00 2001
From: gilch <gilch@users.noreply.github.com>
Date: Wed, 27 Jun 2018 22:37:40 -0600
Subject: [PATCH 026/223] Fix let rebind bug.

---
 hy/contrib/walk.hy                 | 6 ++++--
 tests/native_tests/contrib/walk.hy | 9 +++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index f7d5626..6ee12b2 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -305,6 +305,7 @@ as can nested let forms.
       (macro-error bindings "let bindings must be paired"))
   (setv g!let (gensym 'let)
         replacements (OrderedDict)
+        keys []
         values [])
   (defn expander [symbol]
     (.get replacements symbol symbol))
@@ -315,10 +316,11 @@ as can nested let forms.
                 (macro-error k "binding target may not contain a dot")))
     (.append values (symbolexpand (macroexpand-all v &name)
                                   expander))
-    (assoc replacements k `(get ~g!let ~(name k))))
+    (.append keys `(get ~g!let ~(name k)))
+    (assoc replacements k (last keys)))
   `(do
      (setv ~g!let {}
-           ~@(interleave (.values replacements) values))
+           ~@(interleave keys values))
      ~@(symbolexpand (macroexpand-all body &name)
                      expander)))
 
diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index 2156c3e..730fd45 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -329,3 +329,12 @@
                3))
     (assert (= b 3))
     (assert (= c 3))))
+
+(defn test-let-rebind []
+  (let [x "foo"
+        y "bar"
+        x (+ x y)
+        y (+ y x)
+        x (+ x x)]
+    (assert (= x "foobarfoobar"))
+    (assert (= y "barfoobar"))))

From 9c6714c17637fffa592e76e22a39fd4c90070dfc Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Jun 2018 09:10:44 -0700
Subject: [PATCH 027/223] Remove unused imports

---
 hy/compiler.py    | 4 +---
 hy/importer.py    | 2 +-
 hy/lex/parser.py  | 4 ++--
 tests/test_bin.py | 3 +--
 4 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 04d195d..f72e19c 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -15,8 +15,7 @@ from hy.lex.parser import mangle, unmangle
 
 import hy.macros
 from hy._compat import (
-    str_type, string_types, bytes_type, long_type, PY3, PY35, PY37,
-    raise_empty)
+    str_type, bytes_type, long_type, PY3, PY35, raise_empty)
 from hy.macros import require, macroexpand, tag_macroexpand
 import hy.importer
 
@@ -27,7 +26,6 @@ import sys
 import copy
 
 from collections import defaultdict
-from cmath import isnan
 
 if PY3:
     import builtins
diff --git a/hy/importer.py b/hy/importer.py
index 92c95b0..a93d1fd 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -5,7 +5,7 @@
 from __future__ import absolute_import
 
 from hy.compiler import hy_compile, HyTypeError
-from hy.models import HyObject, HyExpression, HySymbol
+from hy.models import HyExpression, HySymbol
 from hy.lex import tokenize, LexException
 from hy.errors import HyIOError
 
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index fac2577..63ea277 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -6,11 +6,11 @@
 from __future__ import unicode_literals
 
 from functools import wraps
-import string, re, unicodedata
+import re, unicodedata
 
 from rply import ParserGenerator
 
-from hy._compat import PY3, str_type, isidentifier, UCS4
+from hy._compat import str_type, isidentifier, UCS4
 from hy.models import (HyBytes, HyComplex, HyDict, HyExpression, HyFloat,
                        HyInteger, HyKeyword, HyList, HySet, HyString, HySymbol)
 from .lexer import lexer
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 7b34007..1a32e01 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -5,14 +5,13 @@
 # license. See the LICENSE.
 
 import os
-from pipes import quote
 import re
 import shlex
 import subprocess
 
 import pytest
 
-from hy._compat import PY3, PY35, PY36, builtins
+from hy._compat import builtins
 from hy.importer import get_bytecode_path
 
 

From 3d3d1fe6ae5411e1bc39a84e20bde4726a2c22c7 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Jun 2018 09:55:08 -0700
Subject: [PATCH 028/223] Remove unused compiler subroutines

---
 hy/compiler.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index f72e19c..c981f08 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -325,16 +325,6 @@ def _branch(results):
     return ret
 
 
-def _raise_wrong_args_number(expression, error):
-    raise HyTypeError(expression,
-                      error % (expression.pop(0),
-                               len(expression)))
-
-
-def _nargs(n):
-    return "%d argument%s" % (n, ("" if n == 1 else "s"))
-
-
 def is_unpack(kind, x):
     return (isinstance(x, HyExpression)
             and len(x) > 0

From d501b073d87817c548237ddb955e17d6991ed9de Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Jun 2018 09:26:40 -0700
Subject: [PATCH 029/223] Fold compile_time_ns into the compiler

---
 hy/compiler.py | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index c981f08..61c9edb 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -34,17 +34,6 @@ else:
 
 Inf = float('inf')
 
-_compile_time_ns = {}
-
-
-def compile_time_ns(module_name):
-    ns = _compile_time_ns.get(module_name)
-    if ns is None:
-        ns = {'hy': hy, '__name__': module_name}
-        _compile_time_ns[module_name] = ns
-    return ns
-
-
 _stdlib = {}
 
 
@@ -1596,11 +1585,17 @@ class HyASTCompiler(object):
             arg,
             self))
 
+    _namespaces = {}
+
     @special(["eval-and-compile", "eval-when-compile"], [many(FORM)])
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(root)]).replace(expr)
+        if self.module_name not in self._namespaces:
+            # Initialize a compile-time namespace for this module.
+            self._namespaces[self.module_name] = {
+                'hy': hy, '__name__': self.module_name}
         hy.importer.hy_eval(new_expr + body,
-                            compile_time_ns(self.module_name),
+                            self._namespaces[self.module_name],
                             self.module_name)
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"

From 21f7ef0713ed7634399e97f7102278967cd42ef2 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Jun 2018 09:37:14 -0700
Subject: [PATCH 030/223] Fold load_stdlib into the compiler

---
 hy/compiler.py | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 61c9edb..0099d8f 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -34,21 +34,6 @@ else:
 
 Inf = float('inf')
 
-_stdlib = {}
-
-
-def load_stdlib():
-    import hy.core
-    if not _stdlib:
-        for module in hy.core.STDLIB:
-            mod = importlib.import_module(module)
-            for e in map(ast_str, mod.EXPORTS):
-                if getattr(mod, e) is not getattr(builtins, e, ''):
-                    # Don't bother putting a name in _stdlib if it
-                    # points to a builtin with the same name. This
-                    # prevents pointless imports.
-                    _stdlib[e] = module
-
 
 def ast_str(x, piecewise=False):
     if piecewise:
@@ -321,6 +306,9 @@ def is_unpack(kind, x):
             and x[0] == "unpack-" + kind)
 
 
+_stdlib = {}
+
+
 class HyASTCompiler(object):
 
     def __init__(self, module_name):
@@ -333,8 +321,17 @@ class HyASTCompiler(object):
             or module_name == "hy.core.macros")
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
-        if self.can_use_stdlib:
-            load_stdlib()
+        if self.can_use_stdlib and not _stdlib:
+            # Populate _stdlib.
+            import hy.core
+            for module in hy.core.STDLIB:
+                mod = importlib.import_module(module)
+                for e in map(ast_str, mod.EXPORTS):
+                    if getattr(mod, e) is not getattr(builtins, e, ''):
+                        # Don't bother putting a name in _stdlib if it
+                        # points to a builtin with the same name. This
+                        # prevents pointless imports.
+                        _stdlib[e] = module
 
     def get_anon_var(self):
         self.anon_var_count += 1

From 45ec57ab56365f285122bceacb0326982553a7f5 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Jun 2018 09:54:59 -0700
Subject: [PATCH 031/223] Simplify Result.force_expr

---
 hy/compiler.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 0099d8f..7a67aa7 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -184,18 +184,11 @@ class Result(object):
         """
         if self.expr:
             return self.expr
-
-        # Spoof the position of the last statement for our generated None
-        lineno = 0
-        col_offset = 0
-        if self.stmts:
-            lineno = self.stmts[-1].lineno
-            col_offset = self.stmts[-1].col_offset
-
-        return ast.Name(id=ast_str("None"),
-                        ctx=ast.Load(),
-                        lineno=lineno,
-                        col_offset=col_offset)
+        return ast.Name(
+            id=ast_str("None"),
+            ctx=ast.Load(),
+            lineno=self.stmts[-1].lineno if self.stmts else 0,
+            col_offset=self.stmts[-1].col_offset if self.stmts else 0)
 
     def expr_as_stmt(self):
         """Convert the Result's expression context to a statement

From 8a70d5c90fe248c1e303d94005ca5d8a3b8a579e Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Jun 2018 10:10:53 -0700
Subject: [PATCH 032/223] Fold _branch into the compiler

---
 hy/compiler.py | 35 ++++++++++++++---------------------
 1 file changed, 14 insertions(+), 21 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 7a67aa7..36d1c7e 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -272,26 +272,6 @@ class Result(object):
         )
 
 
-def _branch(results):
-    """Make a branch out of a list of Result objects
-
-    This generates a Result from the given sequence of Results, forcing each
-    expression context as a statement before the next result is used.
-
-    We keep the expression context of the last argument for the returned Result
-    """
-    results = list(results)
-    ret = Result()
-    for result in results[:-1]:
-        ret += result
-        ret += result.expr_as_stmt()
-
-    for result in results[-1:]:
-        ret += result
-
-    return ret
-
-
 def is_unpack(kind, x):
     return (isinstance(x, HyExpression)
             and len(x) > 0
@@ -452,7 +432,20 @@ class HyASTCompiler(object):
             return compiled_exprs, ret, keywords
 
     def _compile_branch(self, exprs):
-        return _branch(self.compile(expr) for expr in exprs)
+        """Make a branch out of an iterable of Result objects
+
+        This generates a Result from the given sequence of Results, forcing each
+        expression context as a statement before the next result is used.
+
+        We keep the expression context of the last argument for the returned Result
+        """
+        ret = Result()
+        for x in map(self.compile, exprs[:-1]):
+            ret += x
+            ret += x.expr_as_stmt()
+        if exprs:
+            ret += self.compile(exprs[-1])
+        return ret
 
     def _storeize(self, expr, name, func=None):
         """Return a new `name` object with an ast.Store() context"""

From 217fc2a487395853e53bae5fc0ba5ea2452c98aa Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Jun 2018 16:09:47 -0700
Subject: [PATCH 033/223] Clean up _render_quoted_form

---
 hy/compiler.py | 78 ++++++++++++++++++++++----------------------------
 1 file changed, 35 insertions(+), 43 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 36d1c7e..644ea5e 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -495,63 +495,55 @@ class HyASTCompiler(object):
         We need to distinguish them as want to concatenate them instead of
         just nesting them.
         """
-        if level == 0:
-            if isinstance(form, HyExpression):
-                if form and form[0] in ("unquote", "unquote-splice"):
-                    if len(form) != 2:
-                        raise HyTypeError(form,
-                                          ("`%s' needs 1 argument, got %s" %
-                                           form[0], len(form) - 1))
-                    return set(), form[1], (form[0] == "unquote-splice")
 
-        if isinstance(form, HyExpression):
-            if form and form[0] == "quasiquote":
-                level += 1
-            if form and form[0] in ("unquote", "unquote-splice"):
-                level -= 1
+        op = None
+        if isinstance(form, HyExpression) and form and (
+                isinstance(form[0], HySymbol)):
+            op = unmangle(ast_str(form[0]))
+        if level == 0 and op in ("unquote", "unquote-splice"):
+            if len(form) != 2:
+                raise HyTypeError(form,
+                                  ("`%s' needs 1 argument, got %s" %
+                                   op, len(form) - 1))
+            return set(), form[1], op == "unquote-splice"
+        elif op == "quasiquote":
+            level += 1
+        elif op in ("unquote", "unquote-splice"):
+            level -= 1
 
         name = form.__class__.__name__
         imports = set([name])
+        body = [form]
 
         if isinstance(form, HySequence):
-            if not form:
-                contents = HyList()
-            else:
+            contents = []
+            for x in form:
+                f_imps, f_contents, splice = self._render_quoted_form(x, level)
+                imports.update(f_imps)
+                if splice:
+                    contents.append(HyExpression([
+                        HySymbol("list"),
+                        HyExpression([HySymbol("or"), f_contents, HyList()])]))
+                else:
+                    contents.append(HyList([f_contents]))
+            if form:
                 # If there are arguments, they can be spliced
                 # so we build a sum...
-                contents = HyExpression([HySymbol("+"), HyList()])
-
-            for x in form:
-                f_imports, f_contents, splice = self._render_quoted_form(x,
-                                                                         level)
-                imports.update(f_imports)
-                if splice:
-                    to_add = HyExpression([
-                        HySymbol("list"),
-                        HyExpression([HySymbol("or"), f_contents, HyList()])])
-                else:
-                    to_add = HyList([f_contents])
-
-                contents.append(to_add)
-
-            return imports, HyExpression([HySymbol(name),
-                                          contents]).replace(form), False
+                body = [HyExpression([HySymbol("+"), HyList()] + contents)]
+            else:
+                body = [HyList()]
 
         elif isinstance(form, HySymbol):
-            return imports, HyExpression([HySymbol(name),
-                                          HyString(form)]).replace(form), False
+            body = [HyString(form)]
 
         elif isinstance(form, HyKeyword):
-            return imports, form, False
+            body = [HyString(form.name)]
 
-        elif isinstance(form, HyString):
-            x = [HySymbol(name), form]
-            if form.brackets is not None:
-                x.extend([HyKeyword("brackets"), form.brackets])
-            return imports, HyExpression(x).replace(form), False
+        elif isinstance(form, HyString) and form.brackets is not None:
+            body.extend([HyKeyword("brackets"), form.brackets])
 
-        return imports, HyExpression([HySymbol(name),
-                                      form]).replace(form), False
+        ret = HyExpression([HySymbol(name)] + body).replace(form)
+        return imports, ret, False
 
     @special(["quote", "quasiquote"], [FORM])
     def compile_quote(self, expr, root, arg):

From e2b98effda3921cd30f708ea15852051d52b1aa4 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 19 Jun 2018 16:23:49 -0700
Subject: [PATCH 034/223] Replace an unused variable with `_`

---
 hy/compiler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 644ea5e..2103928 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -548,7 +548,7 @@ class HyASTCompiler(object):
     @special(["quote", "quasiquote"], [FORM])
     def compile_quote(self, expr, root, arg):
         level = Inf if root == "quote" else 0   # Only quasiquotes can unquote
-        imports, stmts, splice = self._render_quoted_form(arg, level)
+        imports, stmts, _ = self._render_quoted_form(arg, level)
         ret = self.compile(stmts)
         ret.add_imports("hy", imports)
         return ret

From 00150c088c0ea6030c38dc0f04c29d2be97c3f9c Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 19 Jun 2018 16:24:02 -0700
Subject: [PATCH 035/223] Remove an unused helper method in the compiler

---
 hy/compiler.py | 26 --------------------------
 1 file changed, 26 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 2103928..a3f2b5c 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -946,32 +946,6 @@ class HyASTCompiler(object):
         elts, ret, _ = self._compile_collect(args)
         return ret + asty.Tuple(expr, elts=elts, ctx=ast.Load())
 
-    def _compile_generator_iterables(self, trailers):
-        """Helper to compile the "trailing" parts of comprehensions:
-        generators and conditions"""
-
-        generators = trailers.pop(0)
-
-        cond = self.compile(trailers.pop(0)) if trailers else Result()
-
-        gen_it = iter(generators)
-        paired_gens = zip(gen_it, gen_it)
-
-        gen_res = Result()
-        gen = []
-        for target, iterable in paired_gens:
-            gen_res += self.compile(iterable)
-            gen.append(ast.comprehension(
-                target=self._storeize(target, self.compile(target)),
-                iter=gen_res.force_expr,
-                ifs=[],
-                is_async=False))
-
-        if cond.expr:
-            gen[-1].ifs.append(cond.expr)
-
-        return gen_res + cond, gen
-
     _loopers = many(
         tag('setv', sym(":setv") + FORM + FORM) |
         tag('if', sym(":if") + FORM) |

From fca2eb93b02d227f476054da829e1e5ff0ebead0 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 19 Jun 2018 16:36:15 -0700
Subject: [PATCH 036/223] Remove dead code from HyASTCompiler.compile

---
 hy/compiler.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index a3f2b5c..8d7c81c 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -348,21 +348,18 @@ class HyASTCompiler(object):
             return Result()
         try:
             ret = self.compile_atom(tree)
-            if ret:
-                self.update_imports(ret)
-                return ret
+            self.update_imports(ret)
+            return ret
         except HyCompileError:
             # compile calls compile, so we're going to have multiple raise
             # nested; so let's re-raise this exception, let's not wrap it in
             # another HyCompileError!
             raise
-        except HyTypeError as e:
+        except HyTypeError:
             raise
         except Exception as e:
             raise_empty(HyCompileError, e, sys.exc_info()[2])
 
-        raise HyCompileError(Exception("Unknown type: `%s'" % _type))
-
     def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
                          oldpy_unpack=False):
         """Collect the expression contexts from a list of compiled expression.

From bd675a5db67ed6b1bbb42da26c1f1b5149945d77 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 21 Jun 2018 10:56:19 -0700
Subject: [PATCH 037/223] Unmangle in compile_expression before build_method

This ensures that e.g. the symbols "~" and "hyx_XtildeX" in the root position will both appear as "~" to the build method.
---
 hy/compiler.py | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 8d7c81c..d6752f2 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -954,7 +954,6 @@ class HyASTCompiler(object):
     @special(["lfor", "sfor", "gfor"], [_loopers, FORM])
     @special(["dfor"], [_loopers, brackets(FORM, FORM)])
     def compile_comprehension(self, expr, root, parts, final):
-        root = unmangle(ast_str(root))
         node_class = {
             "for":  asty.For,
             "lfor": asty.ListComp,
@@ -1136,7 +1135,7 @@ class HyASTCompiler(object):
                 else:
                     assignments = [(k, v or k) for k, v in kids]
 
-            if root == HySymbol("import"):
+            if root == "import":
                 ast_module = ast_str(module, piecewise=True)
                 module = ast_module.lstrip(".")
                 level = len(ast_module) - len(module)
@@ -1158,7 +1157,7 @@ class HyASTCompiler(object):
                         for k, v in assignments]
                 ret += node(
                     expr, module=module or None, names=names, level=level)
-            else: # root == HySymbol("require")
+            else: # root == "require"
                 __import__(module)
                 require(module, self.module_name,
                         assignments=assignments, prefix=prefix)
@@ -1170,8 +1169,9 @@ class HyASTCompiler(object):
         ops = {"and": (ast.And, "True"),
                "or": (ast.Or, "None")}
         opnode, default = ops[operator]
+        osym = expr[0]
         if len(args) == 0:
-            return asty.Name(operator, id=default, ctx=ast.Load())
+            return asty.Name(osym, id=default, ctx=ast.Load())
         elif len(args) == 1:
             return self.compile(args[0])
         ret = Result()
@@ -1179,16 +1179,16 @@ class HyASTCompiler(object):
         if any(value.stmts for value in values):
             # Compile it to an if...else sequence
             var = self.get_anon_var()
-            name = asty.Name(operator, id=var, ctx=ast.Store())
-            expr_name = asty.Name(operator, id=var, ctx=ast.Load())
+            name = asty.Name(osym, id=var, ctx=ast.Store())
+            expr_name = asty.Name(osym, id=var, ctx=ast.Load())
             temp_variables = [name, expr_name]
 
             def make_assign(value, node=None):
                 positioned_name = asty.Name(
-                    node or operator, id=var, ctx=ast.Store())
+                    node or osym, id=var, ctx=ast.Store())
                 temp_variables.append(positioned_name)
                 return asty.Assign(
-                    node or operator, targets=[positioned_name], value=value)
+                    node or osym, targets=[positioned_name], value=value)
 
             current = root = []
             for i, value in enumerate(values):
@@ -1210,7 +1210,7 @@ class HyASTCompiler(object):
             ret = sum(root, ret)
             ret += Result(expr=expr_name, temp_variables=temp_variables)
         else:
-            ret += asty.BoolOp(operator,
+            ret += asty.BoolOp(osym,
                                op=opnode(),
                                values=[value.force_expr for value in values])
         return ret
@@ -1255,8 +1255,6 @@ class HyASTCompiler(object):
     @special(["**", "//", "<<", ">>"], [times(2, Inf, FORM)])
     @special(["%", "^"], [times(2, 2, FORM)])
     def compile_maths_expression(self, expr, root, args):
-        root = unmangle(ast_str(root))
-
         if len(args) == 0:
             # Return the identity element for this operator.
             return asty.Num(expr, n=long_type(
@@ -1293,7 +1291,7 @@ class HyASTCompiler(object):
 
     @special(list(a_ops.keys()), [FORM, FORM])
     def compile_augassign_expression(self, expr, root, target, value):
-        op = self.a_ops[unmangle(ast_str(root))]
+        op = self.a_ops[root]
         target = self._storeize(target, self.compile(target))
         ret = self.compile(value)
         return ret + asty.AugAssign(
@@ -1301,8 +1299,8 @@ class HyASTCompiler(object):
 
     @special("setv", [many(FORM + FORM)])
     def compile_def_expression(self, expr, root, pairs):
-        if len(pairs) == 0:
-            return asty.Name(root, id='None', ctx=ast.Load())
+        if not pairs:
+            return asty.Name(expr, id='None', ctx=ast.Load())
         result = Result()
         for pair in pairs:
             result += self._compile_assign(*pair)
@@ -1535,7 +1533,7 @@ class HyASTCompiler(object):
 
     @special(["eval-and-compile", "eval-when-compile"], [many(FORM)])
     def compile_eval_and_compile(self, expr, root, body):
-        new_expr = HyExpression([HySymbol("do").replace(root)]).replace(expr)
+        new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
         if self.module_name not in self._namespaces:
             # Initialize a compile-time namespace for this module.
             self._namespaces[self.module_name] = {
@@ -1594,7 +1592,7 @@ class HyASTCompiler(object):
                             expression[0],
                             e.msg.replace("<EOF>", "end of form")))
                 return Result() + build_method(
-                    self, expression, expression[0], *parse_tree)
+                    self, expression, unmangle(sfn), *parse_tree)
 
             if fn.startswith("."):
                 # (.split "test test") -> "test test".split()

From 88f33453dc9b6ca38a9e17924096512a5549efd0 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 21 Jun 2018 10:53:21 -0700
Subject: [PATCH 038/223] Minor cleanup for `raise` and `try`

---
 hy/compiler.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index d6752f2..5ece32f 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -575,6 +575,7 @@ class HyASTCompiler(object):
     @special("raise", [maybe(FORM), maybe(sym(":from") + FORM)])
     def compile_raise_expression(self, expr, root, exc, cause):
         ret = Result()
+
         if exc is not None:
             exc = self.compile(exc)
             ret += exc
@@ -587,12 +588,10 @@ class HyASTCompiler(object):
             ret += cause
             cause = cause.force_expr
 
-        ret += asty.Raise(
+        return ret + asty.Raise(
             expr, type=ret.expr, exc=exc,
             inst=None, tback=None, cause=cause)
 
-        return ret
-
     @special("try",
        [many(notpexpr("except", "else", "finally")),
         many(pexpr(sym("except"),
@@ -685,8 +684,6 @@ class HyASTCompiler(object):
         # or
         # []
 
-        # [variable [list of exceptions]]
-        # let's pop variable and use it as name
         name = None
         if len(exceptions) == 2:
             name = exceptions[0]
@@ -697,21 +694,20 @@ class HyASTCompiler(object):
         if isinstance(exceptions_list, HyList):
             if len(exceptions_list):
                 # [FooBar BarFoo] → catch Foobar and BarFoo exceptions
-                elts, _type, _ = self._compile_collect(exceptions_list)
-                _type += asty.Tuple(exceptions_list, elts=elts, ctx=ast.Load())
+                elts, types, _ = self._compile_collect(exceptions_list)
+                types += asty.Tuple(exceptions_list, elts=elts, ctx=ast.Load())
             else:
                 # [] → all exceptions caught
-                _type = Result()
+                types = Result()
         else:
-            _type = self.compile(exceptions_list)
+            types = self.compile(exceptions_list)
 
         body = self._compile_branch(body)
         body += asty.Assign(expr, targets=[var], value=body.force_expr)
         body += body.expr_as_stmt()
 
-        # use _type.expr to get a literal `None`
-        return _type + asty.ExceptHandler(
-            expr, type=_type.expr, name=name,
+        return types + asty.ExceptHandler(
+            expr, type=types.expr, name=name,
             body=body.stmts or [asty.Pass(expr)])
 
     @special("if*", [FORM, FORM, maybe(FORM)])

From af8907b151900f77ec3fa0cee8d90ae880b4fa32 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 21 Jun 2018 11:38:03 -0700
Subject: [PATCH 039/223] Minor cleanup in compile_unary_operator

---
 hy/compiler.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 5ece32f..8fcb854 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1088,12 +1088,9 @@ class HyASTCompiler(object):
         ops = {"not": ast.Not,
                "~": ast.Invert}
         operand = self.compile(arg)
-
-        operand += asty.UnaryOp(
+        return operand + asty.UnaryOp(
             expr, op=ops[root](), operand=operand.force_expr)
 
-        return operand
-
     _symn = some(lambda x: isinstance(x, HySymbol) and "." not in x)
 
     @special(["import", "require"], [many(

From 71dfec9d2f15dcebe3f22ddee8d6013af167fe0a Mon Sep 17 00:00:00 2001
From: Kodi Arfer <Kodiologist@users.noreply.github.com>
Date: Sun, 8 Jul 2018 13:29:08 -0700
Subject: [PATCH 040/223] Add reminder to check master before reporting bugs
 (#1654)

---
 issue_template.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 issue_template.md

diff --git a/issue_template.md b/issue_template.md
new file mode 100644
index 0000000..634a41b
--- /dev/null
+++ b/issue_template.md
@@ -0,0 +1 @@
+If you're reporting a bug, make sure you can reproduce it with the very latest, bleeding-edge version of Hy from the `master` branch on GitHub. Bugs in stable versions of Hy are fixed on `master` before the fix makes it into a new stable release. You can delete this text after reading it.

From 9cc90362d00d3962ca653ef9aeb5d44a17dfadca Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 11 Jul 2018 11:24:19 -0700
Subject: [PATCH 041/223] Docs: string literal prefixes must be in lowercase

---
 docs/language/syntax.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/language/syntax.rst b/docs/language/syntax.rst
index ea089d6..3e28ac6 100644
--- a/docs/language/syntax.rst
+++ b/docs/language/syntax.rst
@@ -67,6 +67,8 @@ of bytes. So when running under Python 3, Hy translates ``"foo"`` and
 ``"foo"`` is translated to ``u"foo"`` and ``b"foo"`` is translated to
 ``"foo"``.
 
+Unlike Python, Hy only recognizes string prefixes (``r``, etc.) in lowercase.
+
 .. _syntax-keywords:
 
 keywords

From 9859b0085cb1bf91f8283ff8ab337bab5c41ec05 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 11 Jul 2018 11:36:32 -0700
Subject: [PATCH 042/223] Document the required order of &-parameters

---
 NEWS.rst              |  2 ++
 docs/language/api.rst | 47 +++++++++++++++++++++++--------------------
 2 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index bd46cb9..24ff925 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -41,6 +41,8 @@ Other Breaking Changes
 * `HySymbol` no longer inherits from `HyString`.
 * `(except)` is no longer allowed. Use `(except [])` instead.
 * `(import [foo])` is no longer allowed. Use `(import foo)` instead.
+* `&`-parameters in lambda lists must now appear in the same order that
+  Python expects.
 
 New Features
 ------------------------------
diff --git a/docs/language/api.rst b/docs/language/api.rst
index 68c9280..e06aaab 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -457,7 +457,11 @@ and the *body* of the function:
 
     (defn name [params] body)
 
-Parameters may have the following keywords in front of them:
+Parameters may be prefixed with the following special symbols. If you use more
+than one, they can only appear in the given order (so all `&optional`
+parameters must precede any `&rest` parameter, `&rest` must precede `&kwonly`,
+and `&kwonly` must precede `&kwargs`). This is the same order that Python
+requires.
 
 &optional
     Parameter is optional. The parameter can be given as a two item list, where
@@ -476,26 +480,6 @@ Parameters may have the following keywords in front of them:
         => (total-value 100 1)
         101.0
 
-&kwargs
-    Parameter will contain 0 or more keyword arguments.
-
-    The following code examples defines a function that will print all keyword
-    arguments and their values.
-
-    .. code-block:: clj
-
-        => (defn print-parameters [&kwargs kwargs]
-        ...    (for [(, k v) (.items kwargs)] (print k v)))
-
-        => (print-parameters :parameter-1 1 :parameter-2 2)
-        parameter_1 1
-        parameter_2 2
-
-        ; to avoid the mangling of '-' to '_', use unpacking:
-        => (print-parameters #** {"parameter-1" 1 "parameter-2" 2})
-        parameter-1 1
-        parameter-2 2
-
 &rest
     Parameter will contain 0 or more positional arguments. No other positional
     arguments may be specified after this one.
@@ -517,7 +501,7 @@ Parameters may have the following keywords in front of them:
         8
         => (zig-zag-sum 1 2 3 4 5 6)
         -3
-
+		
 &kwonly
     .. versionadded:: 0.12.0
 
@@ -553,6 +537,25 @@ Parameters may have the following keywords in front of them:
 
     Availability: Python 3.
 
+&kwargs
+    Parameter will contain 0 or more keyword arguments.
+
+    The following code examples defines a function that will print all keyword
+    arguments and their values.
+
+    .. code-block:: clj
+
+        => (defn print-parameters [&kwargs kwargs]
+        ...    (for [(, k v) (.items kwargs)] (print k v)))
+
+        => (print-parameters :parameter-1 1 :parameter-2 2)
+        parameter_1 1
+        parameter_2 2
+
+        ; to avoid the mangling of '-' to '_', use unpacking:
+        => (print-parameters #** {"parameter-1" 1 "parameter-2" 2})
+        parameter-1 1
+        parameter-2 2
 
 defn/a
 ------

From e05af9d7e0605b986e813cc36f06340773a17726 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 11 Jul 2018 11:59:46 -0700
Subject: [PATCH 043/223] Document function docstrings

---
 docs/language/api.rst | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index e06aaab..cb048a3 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -449,13 +449,16 @@ below:
 defn
 ----
 
-``defn`` macro is used to define functions. It takes three
-parameters: the *name* of the function to define, a vector of *parameters*,
-and the *body* of the function:
+``defn`` is used to define functions. It requires two arguments: a name (given
+as a symbol) and a list of parameters (also given as symbols). Any remaining
+arguments constitute the body of the function.
 
 .. code-block:: clj
 
-    (defn name [params] body)
+    (defn name [params] bodyform1 bodyform2...)
+
+If there at least two body forms, and the first of them is a string literal,
+this string becomes the :ref:`docstring <py:docstring>` of the function.
 
 Parameters may be prefixed with the following special symbols. If you use more
 than one, they can only appear in the given order (so all `&optional`

From abbf29165a62d75702bcf43b32e6a635307d18e4 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sat, 21 Jul 2018 11:20:10 -0700
Subject: [PATCH 044/223] Depend on astor 0.7

---
 .travis.yml | 2 +-
 setup.py    | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 81ea5b8..7a10e06 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,7 @@ python:
   - pypy3
 install:
   - pip install -r requirements-travis.txt
-  - pip install --process-dependency-links -e .
+  - pip install -e .
 script: pytest
 cache: pip
 after_success: make coveralls
diff --git a/setup.py b/setup.py
index 45c5c36..113f53b 100755
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ class Install(install):
                         "." + filename[:-len(".hy")])
         install.run(self)
 
-install_requires = ['rply>=0.7.6', 'astor', 'funcparserlib>=0.3.6', 'clint>=0.4']
+install_requires = ['rply>=0.7.6', 'astor>=0.7.1', 'funcparserlib>=0.3.6', 'clint>=0.4']
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 
@@ -40,9 +40,6 @@ setup(
     name=PKG,
     version=__version__,
     install_requires=install_requires,
-    dependency_links=[
-        'git+https://github.com/berkerpeksag/astor.git#egg=astor-0.7.0'
-    ],
     cmdclass=dict(install=Install),
     entry_points={
         'console_scripts': [

From 9af738e56df0618b914037024c0bf5a7fcf3d507 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 5 Jul 2018 14:06:47 -0700
Subject: [PATCH 045/223] Add hy2py tests for empty data structures

---
 tests/resources/pydemo.hy | 5 +++++
 tests/test_hy2py.py       | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
index b215375..b5e9cd7 100644
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -29,6 +29,11 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little
 (setv myset #{4 5 6})
 (setv mydict {7 8  9 900  10 15})
 
+(setv emptylist [])
+(setv emptytuple (,))
+(setv emptyset #{})
+(setv emptydict {})
+
 (setv mylistcomp (lfor x (range 10) :if (% x 2) x))
 (setv mysetcomp (sfor x (range 5) :if (not (% x 2)) x))
 (setv mydictcomp (dfor k "abcde" :if (!= k "c") [k (.upper k)]))
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
index af3ccab..a0f1188 100644
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -53,6 +53,11 @@ def assert_stuff(m):
     assert m.myset == {4, 5, 6}
     assert m.mydict == {7: 8, 9: 900, 10: 15}
 
+    assert m.emptylist == []
+    assert m.emptytuple == ()
+    assert m.emptyset == set()
+    assert m.emptydict == {}
+
     assert m.mylistcomp == [1, 3, 5, 7, 9]
     assert m.mysetcomp == {0, 2, 4}
     assert m.mydictcomp == dict(a="A", b="B", d="D", e="E")

From 4020f3dd567b047c750daf2cba8a4ce719a27566 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sat, 21 Jul 2018 11:04:07 -0700
Subject: [PATCH 046/223] Clean up NEWS

---
 NEWS.rst | 83 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 41 insertions(+), 42 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 24ff925..3b57b8a 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1,20 +1,20 @@
 .. default-role:: code
 
-Unreleased
+0.15.0
 ==============================
 
 Removals
 ------------------------------
-* Dotted lists, `HyCons`, `cons`, `cons?`, and `list*` have been removed.
-  These were redundant with Python's built-in data structures and Hy's most
-  common model types (`HyExpression`, `HyList`, etc.).
+* Dotted lists, `HyCons`, `cons`, `cons?`, and `list*` have been
+  removed. These were redundant with Python's built-in data structures
+  and Hy's most common model types (`HyExpression`, `HyList`, etc.).
 * `&key` is no longer special in lambda lists. Use `&optional` instead.
-* Tuple unpacking is no longer built into special forms for function
-  definition (`fn` etc.)
-* Macros `ap-pipe` and `ap-compose` have been removed.
-  Anaphoric macros do not work well with point-free style programming,
-  in which case both threading macros and `comp` are more adequate.
+* Lambda lists can no longer unpack tuples.
+* `ap-pipe` and `ap-compose` have been removed. Use threading macros and
+  `comp` instead.
 * `for/a` has been removed. Use `(for [:async ...] ...)` instead.
+* `(except)` is no longer allowed. Use `(except [])` instead.
+* `(import [foo])` is no longer allowed. Use `(import foo)` instead.
 
 Other Breaking Changes
 ------------------------------
@@ -22,58 +22,57 @@ Other Breaking Changes
   This means you can no longer use alternative punctuation in place of
   square brackets in special forms (e.g. `(fn (x) ...)` instead of
   the standard `(fn [x] ...)`).
-* Mangling rules have been overhauled, such that mangled names
-  are always legal Python identifiers
-* `_` and `-` are now equivalent even as single-character names
+* Mangling rules have been overhauled; now, mangled names are
+  always legal Python identifiers.
+* `_` and `-` are now equivalent, even as single-character names.
 
-  * The REPL history variable `_` is now `*1`
+  * The REPL history variable `_` is now `*1`.
 
 * Non-shadow unary `=`, `is`, `<`, etc. now evaluate their argument
-  instead of ignoring it. This change increases consistency a bit
-  and makes accidental unary uses easier to notice.
+  instead of ignoring it.
 * `list-comp`, `set-comp`, `dict-comp`, and `genexpr` have been replaced
   by `lfor`, `sfor`, `dfor`, and `gfor`, respectively, which use a new
   syntax and have additional features. All Python comprehensions can now
   be written in Hy.
-* `hy-repr` uses registered functions instead of methods
-* `HyKeyword` no longer inherits from the string type and has been
-  made into its own object type.
-* `HySymbol` no longer inherits from `HyString`.
-* `(except)` is no longer allowed. Use `(except [])` instead.
-* `(import [foo])` is no longer allowed. Use `(import foo)` instead.
 * `&`-parameters in lambda lists must now appear in the same order that
   Python expects.
+* Literal keywords now evaluate to themselves, and `HyKeyword` no longer
+  inherits from a Python string type
+* `HySymbol` no longer inherits from `HyString`.
 
 New Features
 ------------------------------
-* Python 3.7 is now supported
-* Added `mangle` and `unmangle` as core functions
-* More REPL history result variables: `*2`, `*3`. Added last REPL error
-  variable: `*e`
-* `defclass` in Python 3 now supports specifying metaclasses and other
-  keyword arguments
-* Added a command-line option `-E` per CPython
-* `while` and `for` are allowed to have empty bodies
-* `for` supports the various new clause types offered by `lfor`
-* Added a new module ``hy.model_patterns``
-* `defmacro!` now allows optional args
+* Python 3.7 is now supported.
+* `while` and `for` are allowed to have empty bodies.
+* `for` supports the various new clause types offered by `lfor`.
+* `defclass` in Python 3 supports specifying metaclasses and other
+  keyword arguments.
+* Added `mangle` and `unmangle` as core functions.
+* Added more REPL history variables: `*2` and `*3`.
+* Added a REPL variable holding the last exception: `*e`.
+* Added a command-line option `-E` per CPython.
+* Added a new module `hy.model_patterns`.
 
 Bug Fixes
 ------------------------------
 * `hy2py` should now output legal Python code equivalent to the input Hy
-  code in all cases, modulo some outstanding bugs in `astor`
-* Fix `(return)` so it works correctly to exit a Python 2 generator
-* Fixed a case where `->` and `->>` duplicated an argument
-* Fixed bugs that caused `defclass` to drop statements or crash
-* Fixed a REPL crash caused by illegle unicode escape string inputs
-* `NaN` can no longer create an infinite loop during macro-expansion
-* Fixed a bug that caused `try` to drop expressions
-* Fixed a bug where the compiler didn't properly compile `unquote-splice`
-* Trying to import a dotted name is now a syntax error, as in Python
+  code in all cases.
+* Fixed `(return)` so it can exit a Python 2 generator.
+* Fixed a case where `->` and `->>` duplicated an argument.
+* Fixed bugs that caused `defclass` to drop statements or crash.
+* Fixed a REPL crash caused by illegal backslash escapes.
+* `NaN` can no longer create an infinite loop during macro-expansion.
+* Fixed a bug that caused `try` to drop expressions.
+* The compiler now properly recognizes `unquote-splice`.
+* Trying to import a dotted name is now a syntax error, as in Python.
+* `defmacro!` now allows optional arguments.
+* Fixed handling of variables that are bound multiple times in a single
+  `let`.
 
 Misc. Improvements
 ----------------------------
-* `hy-repr` supports more standard types
+* `hy-repr` uses registered functions instead of methods.
+* `hy-repr` supports more standard types.
 
 0.14.0
 ==============================

From 03aafad65719d147a39cbc948020aa03b5daa94d Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 24 Jul 2018 08:59:52 -0700
Subject: [PATCH 047/223] Make empty expressions illegal at the top level

---
 NEWS.rst                       | 7 +++++++
 hy/compiler.py                 | 5 +++--
 tests/compilers/test_ast.py    | 6 ++++++
 tests/native_tests/language.hy | 5 -----
 4 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 3b57b8a..2b92b99 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1,5 +1,12 @@
 .. default-role:: code
 
+Unreleased
+==============================
+
+Removals
+------------------------------
+* Empty expressions (`()`) are no longer legal at the top level.
+
 0.15.0
 ==============================
 
diff --git a/hy/compiler.py b/hy/compiler.py
index 8fcb854..f92fe4d 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1546,8 +1546,9 @@ class HyASTCompiler(object):
             # Go through compile again if the type changed.
             return self.compile(expression)
 
-        if expression == []:
-            return self.compile_atom(HyList().replace(expression))
+        if not expression:
+            raise HyTypeError(
+                expression, "empty expressions are not allowed at top level")
 
         fn = expression[0]
         func = None
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 5fbc727..a5f42af 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -67,6 +67,12 @@ def test_ast_bad_type():
         pass
 
 
+def test_empty_expr():
+    "Empty expressions should be illegal at the top level."
+    cant_compile("(print ())")
+    can_compile("(print '())")
+
+
 def test_ast_bad_if():
     "Make sure AST can't compile invalid if*"
     cant_compile("(if*)")
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 37f4d4d..59141b5 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1401,11 +1401,6 @@
   (assert (= y [5])))
 
 
-(defn test-empty-list []
-  "Evaluate an empty list to a []"
-  (assert (= () [])))
-
-
 (defn test-string []
   (assert (string? (string "a")))
   (assert (string? (string 1)))

From 1d2c73165dcf6d54308328c3f6e81c349de1bf92 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 24 Jul 2018 09:19:37 -0700
Subject: [PATCH 048/223] Make HyKeyword callable

Co-authored-by: Simon Gomizelj <simon@vodik.xyz>
---
 NEWS.rst                       |  6 ++++++
 docs/language/syntax.rst       |  5 +++++
 hy/compiler.py                 |  7 -------
 hy/models.py                   | 10 ++++++++++
 tests/native_tests/language.hy | 21 ++++++++++++++++++---
 5 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 2b92b99..d4255ef 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -7,6 +7,12 @@ Removals
 ------------------------------
 * Empty expressions (`()`) are no longer legal at the top level.
 
+New Features
+------------------------------
+* Keyword objects (not just literal keywords) can be called, as
+  shorthand for `(get obj :key)`, and they accept a default value
+  as a second argument.
+
 0.15.0
 ==============================
 
diff --git a/docs/language/syntax.rst b/docs/language/syntax.rst
index 3e28ac6..32ebec9 100644
--- a/docs/language/syntax.rst
+++ b/docs/language/syntax.rst
@@ -83,6 +83,11 @@ the error ``Keyword argument :foo needs a value``. To avoid this, you can quote
 the keyword, as in ``(f ':foo)``, or use it as the value of another keyword
 argument, as in ``(f :arg :foo)``.
 
+Keywords can be called like functions as shorthand for ``get``. ``(:foo obj)``
+is equivalent to ``(get obj :foo)``. An optional ``default`` argument is also
+allowed: ``(:foo obj 2)`` or ``(:foo obj :default 2)`` returns ``2`` if ``(get
+obj :foo)`` raises a ``KeyError``.
+
 .. _mangling:
 
 symbols
diff --git a/hy/compiler.py b/hy/compiler.py
index f92fe4d..4f4f694 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1552,13 +1552,6 @@ class HyASTCompiler(object):
 
         fn = expression[0]
         func = None
-        if isinstance(fn, HyKeyword):
-            if len(expression) > 2:
-                raise HyTypeError(
-                    expression, "keyword calls take only 1 argument")
-            expression.append(expression.pop(0))
-            expression.insert(0, HySymbol("get"))
-            return self.compile(expression)
 
         if isinstance(fn, HySymbol):
 
diff --git a/hy/models.py b/hy/models.py
index a0002b6..943458e 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -146,6 +146,16 @@ class HyKeyword(HyObject):
     def __bool__(self):
         return bool(self.name)
 
+    _sentinel = object()
+
+    def __call__(self, data, default=_sentinel):
+        try:
+            return data[self]
+        except KeyError:
+            if default is HyKeyword._sentinel:
+                raise
+            return default
+
 
 def strip_digit_separators(number):
     # Don't strip a _ or , if it's the first character, as _42 and
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 59141b5..3d1c396 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1380,9 +1380,24 @@
   (assert (= (len "ℵℵℵ♥♥♥\t♥♥\r\n") 11)))
 
 
-(defn test-keyword-dict-access []
-  "NATIVE: test keyword dict access"
-  (assert (= "test" (:foo {:foo "test"}))))
+(defn test-keyword-get []
+
+  (assert (= (:foo {:foo "test"}) "test"))
+  (setv f :foo)
+  (assert (= (f {:foo "test"}) "test"))
+
+  (with [(pytest.raises KeyError)] (:foo {:a 1 :b 2}))
+  (assert (= (:foo {:a 1 :b 2} 3) 3))
+  (assert (= (:foo {:a 1 :b 2 :foo 5} 3) 5))
+
+  (with [(pytest.raises TypeError)] (:foo "Hello World"))
+  (with [(pytest.raises TypeError)] (:foo (object)))
+
+  ; The default argument should work regardless of the collection type.
+  (defclass G [object]
+    (defn __getitem__ [self k]
+      (raise KeyError)))
+  (assert (= (:foo (G) 15) 15)))
 
 
 (defn test-break-breaking []

From 45e99d027ded60a413b20c749a65624465a64757 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 12 Jul 2018 14:24:28 -0700
Subject: [PATCH 049/223] Fix an intersphinx link

---
 docs/language/api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index cb048a3..ab0c041 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -458,7 +458,7 @@ arguments constitute the body of the function.
     (defn name [params] bodyform1 bodyform2...)
 
 If there at least two body forms, and the first of them is a string literal,
-this string becomes the :ref:`docstring <py:docstring>` of the function.
+this string becomes the :term:`py:docstring` of the function.
 
 Parameters may be prefixed with the following special symbols. If you use more
 than one, they can only appear in the given order (so all `&optional`

From 271f2846dc255a0d8876c5de4a56b21824b9ceca Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 24 Jul 2018 09:37:19 -0700
Subject: [PATCH 050/223] Minor cleanup in test_ast

---
 tests/compilers/test_ast.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index a5f42af..59232f3 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -16,7 +16,6 @@ from hy._compat import PY3
 import ast
 import pytest
 
-
 def _ast_spotcheck(arg, root, secondary):
     if "." in arg:
         local, full = arg.split(".", 1)
@@ -290,7 +289,7 @@ def test_ast_require():
 
 
 def test_ast_import_require_dotted():
-    """As in Python, it should be a compile-type error to attempt to
+    """As in Python, it should be a compile-time error to attempt to
 import a dotted name."""
     cant_compile("(import [spam [foo.bar]])")
     cant_compile("(require [spam [foo.bar]])")

From 0f85331c81f818922dd07ee19ce216b25f7aaa28 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 24 Jul 2018 09:41:57 -0700
Subject: [PATCH 051/223] Rename variables in @builds_model(HyExpression)

---
 hy/compiler.py | 74 +++++++++++++++++++++++++-------------------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 4f4f694..73395cf 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1539,98 +1539,98 @@ class HyASTCompiler(object):
                 else Result())
 
     @builds_model(HyExpression)
-    def compile_expression(self, expression):
+    def compile_expression(self, expr):
         # Perform macro expansions
-        expression = macroexpand(expression, self)
-        if not isinstance(expression, HyExpression):
+        expr = macroexpand(expr, self)
+        if not isinstance(expr, HyExpression):
             # Go through compile again if the type changed.
-            return self.compile(expression)
+            return self.compile(expr)
 
-        if not expression:
+        if not expr:
             raise HyTypeError(
-                expression, "empty expressions are not allowed at top level")
+                expr, "empty expressions are not allowed at top level")
 
-        fn = expression[0]
+        root = expr[0]
         func = None
 
-        if isinstance(fn, HySymbol):
+        if isinstance(root, HySymbol):
 
-            # First check if `fn` is a special operator, unless it has an
+            # First check if `root` is a special operator, unless it has an
             # `unpack-iterable` in it, since Python's operators (`+`,
             # etc.) can't unpack. An exception to this exception is that
             # tuple literals (`,`) can unpack.
-            sfn = ast_str(fn)
-            if (sfn in _special_form_compilers or sfn in _bad_roots) and (
-                    sfn == mangle(",") or
-                    not any(is_unpack("iterable", x) for x in expression[1:])):
-                if sfn in _bad_roots:
+            sroot = ast_str(root)
+            if (sroot in _special_form_compilers or sroot in _bad_roots) and (
+                    sroot == mangle(",") or
+                    not any(is_unpack("iterable", x) for x in expr[1:])):
+                if sroot in _bad_roots:
                     raise HyTypeError(
-                        expression,
-                        "The special form '{}' is not allowed here".format(fn))
-                # `sfn` is a special operator. Get the build method and
+                        expr,
+                        "The special form '{}' is not allowed here".format(root))
+                # `sroot` is a special operator. Get the build method and
                 # pattern-match the arguments.
-                build_method, pattern = _special_form_compilers[sfn]
+                build_method, pattern = _special_form_compilers[sroot]
                 try:
-                    parse_tree = pattern.parse(expression[1:])
+                    parse_tree = pattern.parse(expr[1:])
                 except NoParseError as e:
                     raise HyTypeError(
-                        expression[min(e.state.pos + 1, len(expression) - 1)],
+                        expr[min(e.state.pos + 1, len(expr) - 1)],
                         "parse error for special form '{}': {}".format(
-                            expression[0],
+                            expr[0],
                             e.msg.replace("<EOF>", "end of form")))
                 return Result() + build_method(
-                    self, expression, unmangle(sfn), *parse_tree)
+                    self, expr, unmangle(sroot), *parse_tree)
 
-            if fn.startswith("."):
+            if root.startswith("."):
                 # (.split "test test") -> "test test".split()
                 # (.a.b.c x) -> (.c (. x a b)) ->  x.a.b.c()
 
                 # Get the method name (the last named attribute
                 # in the chain of attributes)
-                attrs = [HySymbol(a).replace(fn) for a in fn.split(".")[1:]]
-                fn = attrs.pop()
+                attrs = [HySymbol(a).replace(root) for a in root.split(".")[1:]]
+                root = attrs.pop()
 
                 # Get the object we're calling the method on
                 # (extracted with the attribute access DSL)
                 i = 1
-                if len(expression) != 2:
+                if len(expr) != 2:
                     # If the expression has only one object,
                     # always use that as the callee.
                     # Otherwise, hunt for the first thing that
                     # isn't a keyword argument or its value.
-                    while i < len(expression):
-                        if isinstance(expression[i], HyKeyword):
+                    while i < len(expr):
+                        if isinstance(expr[i], HyKeyword):
                             # Skip the keyword argument and its value.
                             i += 1
                         else:
-                            # Use expression[i].
+                            # Use expr[i].
                             break
                         i += 1
                     else:
-                        raise HyTypeError(expression,
+                        raise HyTypeError(expr,
                                           "attribute access requires object")
                 func = self.compile(HyExpression(
-                    [HySymbol(".").replace(fn), expression.pop(i)] +
+                    [HySymbol(".").replace(root), expr.pop(i)] +
                     attrs))
 
                 # And get the method
-                func += asty.Attribute(fn,
+                func += asty.Attribute(root,
                                        value=func.force_expr,
-                                       attr=ast_str(fn),
+                                       attr=ast_str(root),
                                        ctx=ast.Load())
 
         if not func:
-            func = self.compile(fn)
+            func = self.compile(root)
 
         # An exception for pulling together keyword args is if we're doing
         # a typecheck, eg (type :foo)
-        with_kwargs = fn not in (
+        with_kwargs = root not in (
             "type", "HyKeyword", "keyword", "name", "keyword?", "identity")
         args, ret, keywords, oldpy_star, oldpy_kw = self._compile_collect(
-            expression[1:], with_kwargs, oldpy_unpack=True)
+            expr[1:], with_kwargs, oldpy_unpack=True)
 
         return func + ret + asty.Call(
-            expression, func=func.expr, args=args, keywords=keywords,
+            expr, func=func.expr, args=args, keywords=keywords,
             starargs=oldpy_star, kwargs=oldpy_kw)
 
     @builds_model(HyInteger, HyFloat, HyComplex)

From 081a710b0f5fca3bda0b8ffefa627c8b59adbadd Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 24 Jul 2018 09:45:00 -0700
Subject: [PATCH 052/223] Fix handling of unpacking in method calls and
 attribute lookups

---
 NEWS.rst                    |  5 ++++
 hy/compiler.py              | 55 ++++++++++++++++++-------------------
 hy/model_patterns.py        |  9 ++++++
 tests/compilers/test_ast.py | 11 ++++++++
 4 files changed, 52 insertions(+), 28 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index d4255ef..2ec6b16 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -13,6 +13,11 @@ New Features
   shorthand for `(get obj :key)`, and they accept a default value
   as a second argument.
 
+Bug Fixes
+------------------------------
+* Fixed bugs in the handling of unpacking forms in method calls and
+  attribute access.
+
 0.15.0
 ==============================
 
diff --git a/hy/compiler.py b/hy/compiler.py
index 73395cf..7dee28c 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -6,8 +6,8 @@
 from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
                        HyString, HyBytes, HySymbol, HyFloat, HyList, HySet,
                        HyDict, HySequence, wrap_value)
-from hy.model_patterns import (FORM, SYM, STR, sym, brackets, whole, notpexpr,
-                               dolike, pexpr, times, Tag, tag)
+from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole,
+                               notpexpr, dolike, pexpr, times, Tag, tag, unpack)
 from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
 from hy.errors import HyCompileError, HyTypeError
 
@@ -1550,7 +1550,8 @@ class HyASTCompiler(object):
             raise HyTypeError(
                 expr, "empty expressions are not allowed at top level")
 
-        root = expr[0]
+        args = list(expr)
+        root = args.pop(0)
         func = None
 
         if isinstance(root, HySymbol):
@@ -1558,11 +1559,12 @@ class HyASTCompiler(object):
             # First check if `root` is a special operator, unless it has an
             # `unpack-iterable` in it, since Python's operators (`+`,
             # etc.) can't unpack. An exception to this exception is that
-            # tuple literals (`,`) can unpack.
+            # tuple literals (`,`) can unpack. Finally, we allow unpacking in
+            # `.` forms here so the user gets a better error message.
             sroot = ast_str(root)
             if (sroot in _special_form_compilers or sroot in _bad_roots) and (
-                    sroot == mangle(",") or
-                    not any(is_unpack("iterable", x) for x in expr[1:])):
+                    sroot in (mangle(","), mangle(".")) or
+                    not any(is_unpack("iterable", x) for x in args)):
                 if sroot in _bad_roots:
                     raise HyTypeError(
                         expr,
@@ -1571,19 +1573,19 @@ class HyASTCompiler(object):
                 # pattern-match the arguments.
                 build_method, pattern = _special_form_compilers[sroot]
                 try:
-                    parse_tree = pattern.parse(expr[1:])
+                    parse_tree = pattern.parse(args)
                 except NoParseError as e:
                     raise HyTypeError(
                         expr[min(e.state.pos + 1, len(expr) - 1)],
                         "parse error for special form '{}': {}".format(
-                            expr[0],
+                            root,
                             e.msg.replace("<EOF>", "end of form")))
                 return Result() + build_method(
                     self, expr, unmangle(sroot), *parse_tree)
 
             if root.startswith("."):
                 # (.split "test test") -> "test test".split()
-                # (.a.b.c x) -> (.c (. x a b)) ->  x.a.b.c()
+                # (.a.b.c x v1 v2) -> (.c (. x a b) v1 v2) ->  x.a.b.c(v1, v2)
 
                 # Get the method name (the last named attribute
                 # in the chain of attributes)
@@ -1592,25 +1594,22 @@ class HyASTCompiler(object):
 
                 # Get the object we're calling the method on
                 # (extracted with the attribute access DSL)
-                i = 1
-                if len(expr) != 2:
-                    # If the expression has only one object,
-                    # always use that as the callee.
-                    # Otherwise, hunt for the first thing that
-                    # isn't a keyword argument or its value.
-                    while i < len(expr):
-                        if isinstance(expr[i], HyKeyword):
-                            # Skip the keyword argument and its value.
-                            i += 1
-                        else:
-                            # Use expr[i].
-                            break
-                        i += 1
-                    else:
-                        raise HyTypeError(expr,
-                                          "attribute access requires object")
+                # Skip past keywords and their arguments.
+                try:
+                    kws, obj, rest = (
+                        many(KEYWORD + FORM | unpack("mapping")) +
+                        FORM +
+                        many(FORM)).parse(args)
+                except NoParseError:
+                    raise HyTypeError(
+                        expr, "attribute access requires object")
+                # Reconstruct `args` to exclude `obj`.
+                args = [x for p in kws for x in p] + list(rest)
+                if is_unpack("iterable", obj):
+                    raise HyTypeError(
+                        obj, "can't call a method on an unpacking form")
                 func = self.compile(HyExpression(
-                    [HySymbol(".").replace(root), expr.pop(i)] +
+                    [HySymbol(".").replace(root), obj] +
                     attrs))
 
                 # And get the method
@@ -1627,7 +1626,7 @@ class HyASTCompiler(object):
         with_kwargs = root not in (
             "type", "HyKeyword", "keyword", "name", "keyword?", "identity")
         args, ret, keywords, oldpy_star, oldpy_kw = self._compile_collect(
-            expr[1:], with_kwargs, oldpy_unpack=True)
+            args, with_kwargs, oldpy_unpack=True)
 
         return func + ret + asty.Call(
             expr, func=func.expr, args=args, keywords=keywords,
diff --git a/hy/model_patterns.py b/hy/model_patterns.py
index 1d30ccf..5291768 100644
--- a/hy/model_patterns.py
+++ b/hy/model_patterns.py
@@ -15,6 +15,7 @@ from math import isinf
 
 FORM = some(lambda _: True)
 SYM = some(lambda x: isinstance(x, HySymbol))
+KEYWORD = some(lambda x: isinstance(x, HyKeyword))
 STR = some(lambda x: isinstance(x, HyString))
 
 def sym(wanted):
@@ -57,6 +58,14 @@ def notpexpr(*disallowed_heads):
         isinstance(x[0], HySymbol) and
         x[0] in disallowed_heads))
 
+def unpack(kind):
+    "Parse an unpacking form, returning it unchanged."
+    return some(lambda x:
+        isinstance(x, HyExpression)
+        and len(x) > 0
+        and isinstance(x[0], HySymbol)
+        and x[0] == "unpack-" + kind)
+
 def times(lo, hi, parser):
     """Parse `parser` several times (`lo` to `hi`) in a row. `hi` can be
     float('inf'). The result is a list no matter the number of instances."""
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 59232f3..322f78c 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -72,6 +72,17 @@ def test_empty_expr():
     can_compile("(print '())")
 
 
+def test_dot_unpacking():
+
+    can_compile("(.meth obj #* args az)")
+    cant_compile("(.meth #* args az)")
+    cant_compile("(. foo #* bar baz)")
+
+    can_compile("(.meth obj #** args az)")
+    can_compile("(.meth #** args obj)")
+    cant_compile("(. foo #** bar baz)")
+
+
 def test_ast_bad_if():
     "Make sure AST can't compile invalid if*"
     cant_compile("(if*)")

From 33f2b4a91a718a36c8d6118a39bae0c60f09f0bc Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 25 Jul 2018 17:11:32 -0500
Subject: [PATCH 053/223] Compile `require`s in the body of a macro

This change enables further macro expansion for cases in which a macro
`require`s other macros within its body.
---
 hy/contrib/walk.hy | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index 6ee12b2..4d91721 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -43,7 +43,8 @@
 (defn macroexpand-all [form &optional module-name]
   "Recursively performs all possible macroexpansions in form."
   (setv module-name (or module-name (calling-module-name))
-        quote-level [0])  ; TODO: make nonlocal after dropping Python2
+        quote-level [0]
+        ast-compiler (HyASTCompiler module-name))  ; TODO: make nonlocal after dropping Python2
   (defn traverse [form]
     (walk expand identity form))
   (defn expand [form]
@@ -64,7 +65,10 @@
                      [True (traverse form)])]
               [(= (first form) 'quote) form]
               [(= (first form) 'quasiquote) (+quote)]
-              [True (traverse (mexpand form (HyASTCompiler module-name)))])
+              [(= (first form) (HySymbol "require"))
+               (ast-compiler.compile form)
+               (return)]
+              [True (traverse (mexpand form ast-compiler))])
         (if (coll? form)
             (traverse form)
             form)))
@@ -325,4 +329,3 @@ as can nested let forms.
                      expander)))
 
 ;; (defmacro macrolet [])
-

From 65b2bd18ce9e5dd51e237551553ecd8aad05026c Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 25 Jul 2018 17:13:05 -0500
Subject: [PATCH 054/223] Add a test for `require` in the body of a macro

---
 tests/native_tests/contrib/walk.hy | 10 +++++++++-
 tests/resources/macros.hy          |  3 +++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index 730fd45..a93e3bf 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -56,7 +56,15 @@
                 (do '(with [b 2])
                     `(with [c 3]
                        ~(with* [d 4] (do))
-                       ~@[(with* [e 5] (do))]))))))
+                       ~@[(with* [e 5] (do))])))))
+
+  (defmacro require-macro []
+    `(do
+       (require [tests.resources.macros [test-macro :as my-test-macro]])
+       (my-test-macro)))
+
+  (assert (= (last (macroexpand-all '(require-macro)))
+             '(setv blah 1))))
 
 (defn test-let-basic []
   (assert (zero? (let [a 0] a)))
diff --git a/tests/resources/macros.hy b/tests/resources/macros.hy
index 0f5dcbe..300a790 100644
--- a/tests/resources/macros.hy
+++ b/tests/resources/macros.hy
@@ -7,3 +7,6 @@
   (defn f [&rest args] (.join "" (+ (, "c") args)))
   (setv variable (HySymbol (->> "d" (f))))
   `(setv ~variable 5))
+
+(defmacro test-macro []
+  '(setv blah 1))

From a46cc39d6b75d5c0c8d86f2d158bceeb2ff54b3a Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 25 Jul 2018 17:16:17 -0500
Subject: [PATCH 055/223] Include `macroexpand-all` changes in AUTHORS and NEWS

---
 AUTHORS  | 1 +
 NEWS.rst | 1 +
 2 files changed, 2 insertions(+)

diff --git a/AUTHORS b/AUTHORS
index 5ffa1df..0a6c72c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -89,3 +89,4 @@
 * Simon Gomizelj <simon@vodik.xyz>
 * Yigong Wang <wang@yigo.ng>
 * Oskar Kvist <oskar.kvist@gmail.com>
+* Brandon T. Willard <brandonwillard@gmail.com>
\ No newline at end of file
diff --git a/NEWS.rst b/NEWS.rst
index d4255ef..82175d3 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -86,6 +86,7 @@ Misc. Improvements
 ----------------------------
 * `hy-repr` uses registered functions instead of methods.
 * `hy-repr` supports more standard types.
+* `macroexpand-all` will now expand macros introduced by a `require` in the body of a macro.
 
 0.14.0
 ==============================

From 8909ce63cc65a8b0ec5a09594e4b432d58214223 Mon Sep 17 00:00:00 2001
From: gilch <gilch@users.noreply.github.com>
Date: Tue, 31 Jul 2018 20:51:04 -0600
Subject: [PATCH 056/223] Point out appropriate help forums

---
 issue_template.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/issue_template.md b/issue_template.md
index 634a41b..b267f06 100644
--- a/issue_template.md
+++ b/issue_template.md
@@ -1 +1,7 @@
-If you're reporting a bug, make sure you can reproduce it with the very latest, bleeding-edge version of Hy from the `master` branch on GitHub. Bugs in stable versions of Hy are fixed on `master` before the fix makes it into a new stable release. You can delete this text after reading it.
+Hy's issue list is for bugs, complaints, and feature requests.
+
+For help with Hy, ask on our IRC channel (may take up to a day), Stack Overflow with the `[hy]` tag, or on our mailing list.
+
+If you're reporting a bug, make sure you can reproduce it with the very latest, bleeding-edge version of Hy from the `master` branch on GitHub. Bugs in stable versions of Hy are fixed on `master` before the fix makes it into a new stable release.
+
+You can delete this text after reading it.

From 7ba2105a2b861667a732e3fae2ea512528b409fc Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 26 Jul 2018 13:49:04 -0700
Subject: [PATCH 057/223] Fix date and time hy-reprs on Windows

---
 NEWS.rst              |  2 ++
 hy/contrib/hy_repr.hy | 10 +++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index a636f48..5a226f8 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -17,6 +17,8 @@ Bug Fixes
 ------------------------------
 * Fixed bugs in the handling of unpacking forms in method calls and
   attribute access.
+* Fixed crashes on Windows when calling `hy-repr` on date and time
+  objects.
 
 0.15.0
 ==============================
diff --git a/hy/contrib/hy_repr.hy b/hy/contrib/hy_repr.hy
index aca8b2a..5d238ba 100644
--- a/hy/contrib/hy_repr.hy
+++ b/hy/contrib/hy_repr.hy
@@ -121,19 +121,23 @@
 
 (hy-repr-register datetime.datetime (fn [x]
   (.format "(datetime.datetime {}{})"
-    (.strftime x "%Y %-m %-d %-H %-M %-S")
+    (-strftime-0 x "%Y %m %d %H %M %S")
     (-repr-time-innards x))))
 (hy-repr-register datetime.date (fn [x]
-  (.strftime x "(datetime.date %Y %-m %-d)")))
+  (-strftime-0 x "(datetime.date %Y %m %d)")))
 (hy-repr-register datetime.time (fn [x]
   (.format "(datetime.time {}{})"
-    (.strftime x "%-H %-M %-S")
+    (-strftime-0 x "%H %M %S")
     (-repr-time-innards x))))
 (defn -repr-time-innards [x]
   (.rstrip (+ " " (.join " " (filter identity [
     (if x.microsecond (str-type x.microsecond))
     (if (not (none? x.tzinfo)) (+ ":tzinfo " (hy-repr x.tzinfo)))
     (if (and PY36 (!= x.fold 0)) (+ ":fold " (hy-repr x.fold)))])))))
+(defn -strftime-0 [x fmt]
+  ; Remove leading 0s in `strftime`. This is a substitute for the `-`
+  ; flag for when Python isn't built with glibc.
+  (re.sub r"(\A| )0([0-9])" r"\1\2" (.strftime x fmt)))
 
 (hy-repr-register collections.Counter (fn [x]
   (.format "(Counter {})"

From 99851f7f6bd87008d4aa674ce70797cfc8346b69 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 2 Aug 2018 16:27:26 -0400
Subject: [PATCH 058/223] Use fastentrypoints

This speeds up launching `hy`.
---
 fastentrypoints.py | 112 +++++++++++++++++++++++++++++++++++++++++++++
 setup.py           |   1 +
 2 files changed, 113 insertions(+)
 create mode 100644 fastentrypoints.py

diff --git a/fastentrypoints.py b/fastentrypoints.py
new file mode 100644
index 0000000..9707f74
--- /dev/null
+++ b/fastentrypoints.py
@@ -0,0 +1,112 @@
+# noqa: D300,D400
+# Copyright (c) 2016, Aaron Christianson
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+Monkey patch setuptools to write faster console_scripts with this format:
+
+    import sys
+    from mymodule import entry_function
+    sys.exit(entry_function())
+
+This is better.
+
+(c) 2016, Aaron Christianson
+http://github.com/ninjaaron/fast-entry_points
+'''
+from setuptools.command import easy_install
+import re
+TEMPLATE = '''\
+# -*- coding: utf-8 -*-
+# EASY-INSTALL-ENTRY-SCRIPT: '{3}','{4}','{5}'
+__requires__ = '{3}'
+import re
+import sys
+
+from {0} import {1}
+
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+    sys.exit({2}())'''
+
+
+@classmethod
+def get_args(cls, dist, header=None):  # noqa: D205,D400
+    """
+    Yield write_script() argument tuples for a distribution's
+    console_scripts and gui_scripts entry points.
+    """
+    if header is None:
+        # pylint: disable=E1101
+        header = cls.get_header()
+    spec = str(dist.as_requirement())
+    for type_ in 'console', 'gui':
+        group = type_ + '_scripts'
+        for name, ep in dist.get_entry_map(group).items():
+            # ensure_safe_name
+            if re.search(r'[\\/]', name):
+                raise ValueError("Path separators not allowed in script names")
+            script_text = TEMPLATE.format(
+                ep.module_name, ep.attrs[0], '.'.join(ep.attrs),
+                spec, group, name)
+            # pylint: disable=E1101
+            args = cls._get_script_args(type_, name, header, script_text)
+            for res in args:
+                yield res
+
+
+# pylint: disable=E1101
+easy_install.ScriptWriter.get_args = get_args
+
+
+def main():
+    import os
+    import re
+    import shutil
+    import sys
+    dests = sys.argv[1:] or ['.']
+    filename = re.sub('\.pyc$', '.py', __file__)
+
+    for dst in dests:
+        shutil.copy(filename, dst)
+        manifest_path = os.path.join(dst, 'MANIFEST.in')
+        setup_path = os.path.join(dst, 'setup.py')
+
+        # Insert the include statement to MANIFEST.in if not present
+        with open(manifest_path, 'a+') as manifest:
+            manifest.seek(0)
+            manifest_content = manifest.read()
+            if 'include fastentrypoints.py' not in manifest_content:
+                manifest.write(('\n' if manifest_content else '') +
+                               'include fastentrypoints.py')
+
+        # Insert the import statement to setup.py if not present
+        with open(setup_path, 'a+') as setup:
+            setup.seek(0)
+            setup_content = setup.read()
+            if 'import fastentrypoints' not in setup_content:
+                setup.seek(0)
+                setup.truncate()
+                setup.write('import fastentrypoints\n' + setup_content)
diff --git a/setup.py b/setup.py
index 113f53b..a8a28ca 100755
--- a/setup.py
+++ b/setup.py
@@ -7,6 +7,7 @@ import sys, os
 
 from setuptools import find_packages, setup
 from setuptools.command.install import install
+import fastentrypoints   # Monkey-patches setuptools.
 
 from get_version import __version__
 

From 734cdcd2fd760febc003b56769bcd2d0c12d6960 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 2 Aug 2018 20:20:42 -0400
Subject: [PATCH 059/223] Delay importing the lexer and parser

This speeds up runs of Hy that never need to parse or compile Hy code (e.g., running a Hy program that's already byte-compiled).
---
 hy/cmdline.py       |  3 +-
 hy/compiler.py      |  2 +-
 hy/core/language.hy |  3 +-
 hy/lex/__init__.py  | 87 ++++++++++++++++++++++++++++++++++++++++++---
 hy/lex/parser.py    | 76 +--------------------------------------
 hy/macros.py        |  2 +-
 6 files changed, 88 insertions(+), 85 deletions(-)

diff --git a/hy/cmdline.py b/hy/cmdline.py
index 3f2bb1f..b10028c 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -15,8 +15,7 @@ import astor.code_gen
 
 import hy
 
-from hy.lex import LexException, PrematureEndOfInput
-from hy.lex.parser import mangle
+from hy.lex import LexException, PrematureEndOfInput, mangle
 from hy.compiler import HyTypeError
 from hy.importer import (hy_eval, import_buffer_to_module,
                          import_file_to_ast, import_file_to_hst,
diff --git a/hy/compiler.py b/hy/compiler.py
index 7dee28c..f25f615 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -11,7 +11,7 @@ from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole,
 from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
 from hy.errors import HyCompileError, HyTypeError
 
-from hy.lex.parser import mangle, unmangle
+from hy.lex import mangle, unmangle
 
 import hy.macros
 from hy._compat import (
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 2895bb7..abb387f 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -19,8 +19,7 @@
   (import [collections :as cabc])
   (import [collections.abc :as cabc]))
 (import [hy.models [HySymbol HyKeyword]])
-(import [hy.lex [LexException PrematureEndOfInput tokenize]])
-(import [hy.lex.parser [mangle unmangle]])
+(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
 (import [hy.compiler [HyASTCompiler]])
 (import [hy.importer [hy-eval :as eval]])
 
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index 85d6502..5c05143 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -2,17 +2,19 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
-from rply.errors import LexingError
+from __future__ import unicode_literals
 
+import re, unicodedata
+from hy._compat import str_type, isidentifier, UCS4
 from hy.lex.exceptions import LexException, PrematureEndOfInput  # NOQA
-from hy.lex.lexer import lexer
-from hy.lex.parser import parser
-
 
 def tokenize(buf):
     """
     Tokenize a Lisp file or string buffer into internal Hy objects.
     """
+    from hy.lex.lexer import lexer
+    from hy.lex.parser import parser
+    from rply.errors import LexingError
     try:
         return parser.parse(lexer.lex(buf))
     except LexingError as e:
@@ -23,3 +25,80 @@ def tokenize(buf):
         if e.source is None:
             e.source = buf
         raise
+
+
+mangle_delim = 'X'
+
+
+def mangle(s):
+    """Stringify the argument and convert it to a valid Python identifier
+    according to Hy's mangling rules."""
+    def unicode_char_to_hex(uchr):
+        # Covert a unicode char to hex string, without prefix
+        return uchr.encode('unicode-escape').decode('utf-8').lstrip('\\U').lstrip('\\u').lstrip('0')
+
+    assert s
+
+    s = str_type(s)
+    s = s.replace("-", "_")
+    s2 = s.lstrip('_')
+    leading_underscores = '_' * (len(s) - len(s2))
+    s = s2
+
+    if s.endswith("?"):
+        s = 'is_' + s[:-1]
+    if not isidentifier(leading_underscores + s):
+        # Replace illegal characters with their Unicode character
+        # names, or hexadecimal if they don't have one.
+        s = 'hyx_' + ''.join(
+            c
+               if c != mangle_delim and isidentifier('S' + c)
+                 # We prepend the "S" because some characters aren't
+                 # allowed at the start of an identifier.
+               else '{0}{1}{0}'.format(mangle_delim,
+                   unicodedata.name(c, '').lower().replace('-', 'H').replace(' ', '_')
+                   or 'U{}'.format(unicode_char_to_hex(c)))
+            for c in unicode_to_ucs4iter(s))
+
+    s = leading_underscores + s
+    assert isidentifier(s)
+    return s
+
+
+def unmangle(s):
+    """Stringify the argument and try to convert it to a pretty unmangled
+    form. This may not round-trip, because different Hy symbol names can
+    mangle to the same Python identifier."""
+
+    s = str_type(s)
+
+    s2 = s.lstrip('_')
+    leading_underscores = len(s) - len(s2)
+    s = s2
+
+    if s.startswith('hyx_'):
+        s = re.sub('{0}(U)?([_a-z0-9H]+?){0}'.format(mangle_delim),
+            lambda mo:
+               chr(int(mo.group(2), base=16))
+               if mo.group(1)
+               else unicodedata.lookup(
+                   mo.group(2).replace('_', ' ').replace('H', '-').upper()),
+            s[len('hyx_'):])
+    if s.startswith('is_'):
+        s = s[len("is_"):] + "?"
+    s = s.replace('_', '-')
+
+    return '-' * leading_underscores + s
+
+
+def unicode_to_ucs4iter(ustr):
+    # Covert a unicode string to an iterable object,
+    # elements in the object are single USC-4 unicode characters
+    if UCS4:
+        return ustr
+    ucs4_list = list(ustr)
+    for i, u in enumerate(ucs4_list):
+        if 0xD7FF < ord(u) < 0xDC00:
+            ucs4_list[i] += ucs4_list[i + 1]
+            del ucs4_list[i + 1]
+    return ucs4_list
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index 63ea277..a13c793 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -10,7 +10,7 @@ import re, unicodedata
 
 from rply import ParserGenerator
 
-from hy._compat import str_type, isidentifier, UCS4
+from hy._compat import str_type
 from hy.models import (HyBytes, HyComplex, HyDict, HyExpression, HyFloat,
                        HyInteger, HyKeyword, HyList, HySet, HyString, HySymbol)
 from .lexer import lexer
@@ -19,80 +19,6 @@ from .exceptions import LexException, PrematureEndOfInput
 
 pg = ParserGenerator([rule.name for rule in lexer.rules] + ['$end'])
 
-mangle_delim = 'X'
-
-def unicode_to_ucs4iter(ustr):
-    # Covert a unicode string to an iterable object,
-    # elements in the object are single USC-4 unicode characters
-    if UCS4:
-        return ustr
-    ucs4_list = list(ustr)
-    for i, u in enumerate(ucs4_list):
-        if 0xD7FF < ord(u) < 0xDC00:
-            ucs4_list[i] += ucs4_list[i + 1]
-            del ucs4_list[i + 1]
-    return ucs4_list
-
-def mangle(s):
-    """Stringify the argument and convert it to a valid Python identifier
-    according to Hy's mangling rules."""
-    def unicode_char_to_hex(uchr):
-        # Covert a unicode char to hex string, without prefix
-        return uchr.encode('unicode-escape').decode('utf-8').lstrip('\\U').lstrip('\\u').lstrip('0')
-
-    assert s
-
-    s = str_type(s)
-    s = s.replace("-", "_")
-    s2 = s.lstrip('_')
-    leading_underscores = '_' * (len(s) - len(s2))
-    s = s2
-
-    if s.endswith("?"):
-        s = 'is_' + s[:-1]
-    if not isidentifier(leading_underscores + s):
-        # Replace illegal characters with their Unicode character
-        # names, or hexadecimal if they don't have one.
-        s = 'hyx_' + ''.join(
-            c
-               if c != mangle_delim and isidentifier('S' + c)
-                 # We prepend the "S" because some characters aren't
-                 # allowed at the start of an identifier.
-               else '{0}{1}{0}'.format(mangle_delim,
-                   unicodedata.name(c, '').lower().replace('-', 'H').replace(' ', '_')
-                   or 'U{}'.format(unicode_char_to_hex(c)))
-            for c in unicode_to_ucs4iter(s))
-
-    s = leading_underscores + s
-    assert isidentifier(s)
-    return s
-
-
-def unmangle(s):
-    """Stringify the argument and try to convert it to a pretty unmangled
-    form. This may not round-trip, because different Hy symbol names can
-    mangle to the same Python identifier."""
-
-    s = str_type(s)
-
-    s2 = s.lstrip('_')
-    leading_underscores = len(s) - len(s2)
-    s = s2
-
-    if s.startswith('hyx_'):
-        s = re.sub('{0}(U)?([_a-z0-9H]+?){0}'.format(mangle_delim),
-            lambda mo:
-               chr(int(mo.group(2), base=16))
-               if mo.group(1)
-               else unicodedata.lookup(
-                   mo.group(2).replace('_', ' ').replace('H', '-').upper()),
-            s[len('hyx_'):])
-    if s.startswith('is_'):
-        s = s[len("is_"):] + "?"
-    s = s.replace('_', '-')
-
-    return '-' * leading_underscores + s
-
 
 def set_boundaries(fun):
     @wraps(fun)
diff --git a/hy/macros.py b/hy/macros.py
index 52702ee..c5ab844 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -5,7 +5,7 @@
 from hy._compat import PY3
 import hy.inspect
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
-from hy.lex.parser import mangle
+from hy.lex import mangle
 from hy._compat import str_type
 
 from hy.errors import HyTypeError, HyMacroExpansionError

From 87a5b117a14f09f3ca0b35135d1e3c175b124bae Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sun, 19 Aug 2018 23:29:29 -0500
Subject: [PATCH 060/223] Implement new importer using PEP-302 semantics

Python 3.x is patched in a way that integrates `.hy` source files into
Pythons default `importlib` machinery.  In Python 2.7, a PEP-302 "importer"
and "loader" is implemented according to the standard `import` logic (via
`pkgutil` and later pure-Python `imp` package code).

In both cases, the entry-point for the loaders is through `sys.path_hooks` only.
As well, the import semantics have been updated all throughout to utilize
`importlib` and follow aspects of PEP-420.  This, along with some light
patches, should allow for basic use of `runpy`, `py_compile` and `reload`.

In all cases, if a `.hy` file is shadowed by a `.py`, Hy will silently use
`.hy`.
---
 conftest.py                         |  49 +-
 docs/contrib/walk.rst               |   2 +-
 hy/_compat.py                       |  17 +-
 hy/cmdline.py                       | 117 +++--
 hy/compiler.py                      |   2 +-
 hy/contrib/walk.hy                  |   2 +-
 hy/core/macros.hy                   |   9 +-
 hy/importer.py                      | 705 +++++++++++++++++++---------
 hy/macros.py                        |  20 +-
 tests/compilers/test_ast.py         |  10 +-
 tests/importer/test_importer.py     |  54 ++-
 tests/importer/test_pyc.py          |  21 +-
 tests/native_tests/native_macros.hy |  28 +-
 tests/test_bin.py                   |  26 +-
 14 files changed, 691 insertions(+), 371 deletions(-)

diff --git a/conftest.py b/conftest.py
index 54ba2e6..5b1c41d 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,15 +1,50 @@
+import os
+import importlib
+
+import py
 import pytest
 import hy
-import os
 from hy._compat import PY3, PY35, PY36
 
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
+_fspath_pyimport = py.path.local.pyimport
+
+
+def pytest_ignore_collect(path, config):
+    return (("py3_only" in path.basename and not PY3)
+            or ("py35_only" in path.basename and not PY35)
+            or ("py36_only" in path.basename and not PY36))
+
+
+def pyimport_patch_mismatch(self, **kwargs):
+    """Lame fix for https://github.com/pytest-dev/py/issues/195"""
+    try:
+        return _fspath_pyimport(self, **kwargs)
+    except py.path.local.ImportMismatchError:
+        pkgpath = self.pypkgpath()
+        if pkgpath is None:
+            pkgroot = self.dirpath()
+            modname = self.purebasename
+        else:
+            pkgroot = pkgpath.dirpath()
+            names = self.new(ext="").relto(pkgroot).split(self.sep)
+            if names[-1] == "__init__":
+                names.pop()
+            modname = ".".join(names)
+
+        res = importlib.import_module(modname)
+
+        return res
+
+
+py.path.local.pyimport = pyimport_patch_mismatch
+
+
 def pytest_collect_file(parent, path):
     if (path.ext == ".hy"
-            and NATIVE_TESTS in path.dirname + os.sep
-            and path.basename != "__init__.hy"
-            and not ("py3_only" in path.basename and not PY3)
-            and not ("py35_only" in path.basename and not PY35)
-            and not ("py36_only" in path.basename and not PY36)):
-        return pytest.Module(path, parent)
+        and NATIVE_TESTS in path.dirname + os.sep
+        and path.basename != "__init__.hy"):
+
+        pytest_mod = pytest.Module(path, parent)
+        return pytest_mod
diff --git a/docs/contrib/walk.rst b/docs/contrib/walk.rst
index 1bd0fcc..2e36234 100644
--- a/docs/contrib/walk.rst
+++ b/docs/contrib/walk.rst
@@ -231,7 +231,7 @@ But assignments via ``import`` are always hoisted to normal Python scope, and
 likewise, ``defclass`` will assign the class to the Python scope,
 even if it shares the name of a let binding.
 
-Use ``__import__`` and ``type`` (or whatever metaclass) instead,
+Use ``importlib.import_module`` and ``type`` (or whatever metaclass) instead,
 if you must avoid this hoisting.
 
 The ``let`` macro takes two parameters: a list defining *variables*
diff --git a/hy/_compat.py b/hy/_compat.py
index c40e44d..4416ea5 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -6,18 +6,6 @@ try:
     import __builtin__ as builtins
 except ImportError:
     import builtins  # NOQA
-try:
-    from py_compile import MAGIC, wr_long
-except ImportError:
-    # py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4
-    from importlib.util import MAGIC_NUMBER as MAGIC  # NOQA
-
-    def wr_long(f, x):
-        """Internal; write a 32-bit int to a file in little-endian order."""
-        f.write(bytes([x & 0xff,
-                       (x >> 8) & 0xff,
-                       (x >> 16) & 0xff,
-                       (x >> 24) & 0xff]))
 import sys, keyword
 
 PY3 = sys.version_info[0] >= 3
@@ -60,3 +48,8 @@ def isidentifier(x):
     except T.TokenError:
         return False
     return len(tokens) == 2 and tokens[0][0] == T.NAME
+
+try:
+    FileNotFoundError = FileNotFoundError
+except NameError:
+    FileNotFoundError = IOError
diff --git a/hy/cmdline.py b/hy/cmdline.py
index b10028c..f1c7a9e 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -9,26 +9,21 @@ import code
 import ast
 import sys
 import os
+import io
 import importlib
+import py_compile
+import runpy
 
 import astor.code_gen
 
 import hy
-
 from hy.lex import LexException, PrematureEndOfInput, mangle
-from hy.compiler import HyTypeError
-from hy.importer import (hy_eval, import_buffer_to_module,
-                         import_file_to_ast, import_file_to_hst,
-                         import_buffer_to_ast, import_buffer_to_hst)
-from hy.completer import completion
-from hy.completer import Completer
-
-from hy.errors import HyIOError
-
+from hy.compiler import HyTypeError, hy_compile
+from hy.importer import hy_eval, hy_parse
+from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
-
-from hy._compat import builtins, PY3
+from hy._compat import builtins, PY3, FileNotFoundError
 
 
 class HyQuitter(object):
@@ -47,6 +42,7 @@ class HyQuitter(object):
             pass
         raise SystemExit(code)
 
+
 builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
@@ -88,7 +84,7 @@ class HyREPL(code.InteractiveConsole):
 
         try:
             try:
-                do = import_buffer_to_hst(source)
+                do = hy_parse(source)
             except PrematureEndOfInput:
                 return True
         except LexException as e:
@@ -202,25 +198,8 @@ def pretty_error(func, *args, **kw):
 
 
 def run_command(source):
-    pretty_error(import_buffer_to_module, "__main__", source)
-    return 0
-
-
-def run_module(mod_name):
-    from hy.importer import MetaImporter
-    pth = MetaImporter().find_on_path(mod_name)
-    if pth is not None:
-        sys.argv = [pth] + sys.argv
-        return run_file(pth)
-
-    print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name),
-          file=sys.stderr)
-    return 1
-
-
-def run_file(filename):
-    from hy.importer import import_file_to_module
-    pretty_error(import_file_to_module, "__main__", filename)
+    tree = hy_parse(source)
+    pretty_error(hy_eval, tree, module_name="__main__")
     return 0
 
 
@@ -252,7 +231,7 @@ def run_repl(hr=None, **kwargs):
 def run_icommand(source, **kwargs):
     hr = HyREPL(**kwargs)
     if os.path.exists(source):
-        with open(source, "r") as f:
+        with io.open(source, "r", encoding='utf-8') as f:
             source = f.read()
         filename = source
     else:
@@ -320,7 +299,7 @@ def cmdline_handler(scriptname, argv):
         SIMPLE_TRACEBACKS = False
 
     # reset sys.argv like Python
-    sys.argv = options.args + module_args or [""]
+    # sys.argv = [sys.argv[0]] + options.args + module_args
 
     if options.E:
         # User did "hy -E ..."
@@ -332,7 +311,9 @@ def cmdline_handler(scriptname, argv):
 
     if options.mod:
         # User did "hy -m ..."
-        return run_module(options.mod)
+        sys.argv = [sys.argv[0]] + options.args + module_args
+        runpy.run_module(options.mod, run_name='__main__', alter_sys=True)
+        return 0
 
     if options.icommand:
         # User did "hy -i ..."
@@ -347,10 +328,12 @@ def cmdline_handler(scriptname, argv):
         else:
             # User did "hy <filename>"
             try:
-                return run_file(options.args[0])
-            except HyIOError as e:
-                print("hy: Can't open file '{0}': [Errno {1}] {2}\n".format(
-                    e.filename, e.errno, e.strerror), file=sys.stderr)
+                sys.argv = options.args
+                runpy.run_path(options.args[0], run_name='__main__')
+                return 0
+            except FileNotFoundError as e:
+                print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
+                      e.filename, e.errno, e.strerror), file=sys.stderr)
                 sys.exit(e.errno)
 
     # User did NOTHING!
@@ -359,27 +342,45 @@ def cmdline_handler(scriptname, argv):
 
 # entry point for cmd line script "hy"
 def hy_main():
+    sys.path.insert(0, "")
     sys.exit(cmdline_handler("hy", sys.argv))
 
 
-# entry point for cmd line script "hyc"
 def hyc_main():
-    from hy.importer import write_hy_as_pyc
     parser = argparse.ArgumentParser(prog="hyc")
-    parser.add_argument("files", metavar="FILE", nargs='+',
-                        help="file to compile")
+    parser.add_argument("files", metavar="FILE", nargs='*',
+                        help=('File(s) to compile (use STDIN if only'
+                              ' "-" or nothing is provided)'))
     parser.add_argument("-v", action="version", version=VERSION)
 
     options = parser.parse_args(sys.argv[1:])
 
-    for file in options.files:
-        try:
-            print("Compiling %s" % file)
-            pretty_error(write_hy_as_pyc, file)
-        except IOError as x:
-            print("hyc: Can't open file '{0}': [Errno {1}] {2}\n".format(
-                x.filename, x.errno, x.strerror), file=sys.stderr)
-            sys.exit(x.errno)
+    rv = 0
+    if len(options.files) == 0 or (
+            len(options.files) == 1 and options.files[0] == '-'):
+        while True:
+            filename = sys.stdin.readline()
+            if not filename:
+                break
+            filename = filename.rstrip('\n')
+            try:
+                py_compile.compile(filename, doraise=True)
+            except py_compile.PyCompileError as error:
+                rv = 1
+                sys.stderr.write("%s\n" % error.msg)
+            except OSError as error:
+                rv = 1
+                sys.stderr.write("%s\n" % error)
+    else:
+        for filename in options.files:
+            try:
+                print("Compiling %s" % filename)
+                py_compile.compile(filename, doraise=True)
+            except py_compile.PyCompileError as error:
+                # return value to indicate at least one failure
+                rv = 1
+                sys.stderr.write("%s\n" % error.msg)
+    return rv
 
 
 # entry point for cmd line script "hy2py"
@@ -403,14 +404,14 @@ def hy2py_main():
 
     options = parser.parse_args(sys.argv[1:])
 
-    stdin_text = None
     if options.FILE is None or options.FILE == '-':
-        stdin_text = sys.stdin.read()
+        source = sys.stdin.read()
+    else:
+        with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
+            source = source_file.read()
 
+    hst = pretty_error(hy_parse, source)
     if options.with_source:
-        hst = (pretty_error(import_file_to_hst, options.FILE)
-               if stdin_text is None
-               else pretty_error(import_buffer_to_hst, stdin_text))
         # need special printing on Windows in case the
         # codepage doesn't support utf-8 characters
         if PY3 and platform.system() == "Windows":
@@ -424,9 +425,7 @@ def hy2py_main():
         print()
         print()
 
-    _ast = (pretty_error(import_file_to_ast, options.FILE, module_name)
-            if stdin_text is None
-            else pretty_error(import_buffer_to_ast, stdin_text, module_name))
+    _ast = pretty_error(hy_compile, hst, module_name)
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))
diff --git a/hy/compiler.py b/hy/compiler.py
index f25f615..575c5a9 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1151,7 +1151,7 @@ class HyASTCompiler(object):
                 ret += node(
                     expr, module=module or None, names=names, level=level)
             else: # root == "require"
-                __import__(module)
+                importlib.import_module(module)
                 require(module, self.module_name,
                         assignments=assignments, prefix=prefix)
 
diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index 4d91721..4b6df7a 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -299,7 +299,7 @@ But assignments via `import` are always hoisted to normal Python scope, and
 likewise, `defclass` will assign the class to the Python scope,
 even if it shares the name of a let binding.
 
-Use __import__ and type (or whatever metaclass) instead,
+Use `import_module` and `type` (or whatever metaclass) instead,
 if you must avoid this hoisting.
 
 Function arguments can shadow let bindings in their body,
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index c607c74..744e237 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -6,6 +6,7 @@
 ;;; These macros form the hy language
 ;;; They are automatically required in every module, except inside hy.core
 
+(import [importlib [import-module]])
 
 (import [hy.models [HyList HySymbol]])
 
@@ -247,13 +248,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
    Use ``#doc foo`` instead for help with tag macro ``#foo``.
    Use ``(help foo)`` instead for help with runtime objects."
   `(try
-     (help (. (__import__ "hy")
+     (help (. (import-module "hy")
               macros
               _hy_macros
               [__name__]
               ['~symbol]))
      (except [KeyError]
-       (help (. (__import__ "hy")
+       (help (. (import-module "hy")
                 macros
                 _hy_macros
                 [None]
@@ -264,13 +265,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
 
    Gets help for a tag macro function available in this module."
   `(try
-     (help (. (__import__ "hy")
+     (help (. (import-module "hy")
               macros
               _hy_tag
               [__name__]
               ['~symbol]))
      (except [KeyError]
-       (help (. (__import__ "hy")
+       (help (. (import-module "hy")
                 macros
                 _hy_tag
                 [None]
diff --git a/hy/importer.py b/hy/importer.py
index a93d1fd..390f373 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -4,158 +4,75 @@
 
 from __future__ import absolute_import
 
-from hy.compiler import hy_compile, HyTypeError
-from hy.models import HyExpression, HySymbol
-from hy.lex import tokenize, LexException
-from hy.errors import HyIOError
-
-from io import open
-import re
-import marshal
-import struct
-import imp
 import sys
+import os
 import ast
 import inspect
-import os
+import pkgutil
+import re
+import io
+import runpy
+import types
+import tempfile
+import importlib
 import __future__
 
-from hy._compat import PY3, PY37, MAGIC, builtins, long_type, wr_long
-from hy._compat import string_types
+from hy.errors import HyTypeError
+from hy.compiler import hy_compile
+from hy.lex import tokenize, LexException
+from hy.models import HyExpression, HySymbol
+from hy._compat import string_types, PY3
+
+
+hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
+                        __future__.CO_FUTURE_PRINT_FUNCTION)
 
 
 def ast_compile(ast, filename, mode):
     """Compile AST.
-    Like Python's compile, but with some special flags."""
-    flags = (__future__.CO_FUTURE_DIVISION |
-             __future__.CO_FUTURE_PRINT_FUNCTION)
-    return compile(ast, filename, mode, flags)
+
+    Parameters
+    ----------
+    ast : instance of `ast.AST`
+
+    filename : str
+        Filename used for run-time error messages
+
+    mode: str
+        `compile` mode parameter
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    return compile(ast, filename, mode, hy_ast_compile_flags)
 
 
-def import_buffer_to_hst(buf):
-    """Import content from buf and return a Hy AST."""
-    return HyExpression([HySymbol("do")] + tokenize(buf + "\n"))
+def hy_parse(source):
+    """Parse a Hy source string.
 
+    Parameters
+    ----------
+    source: string
+        Source code to parse.
 
-def import_file_to_hst(fpath):
-    """Import content from fpath and return a Hy AST."""
-    try:
-        with open(fpath, 'r', encoding='utf-8') as f:
-            buf = f.read()
-        # Strip the shebang line, if there is one.
-        buf = re.sub(r'\A#!.*', '', buf)
-        return import_buffer_to_hst(buf)
-    except IOError as e:
-        raise HyIOError(e.errno, e.strerror, e.filename)
-
-
-def import_buffer_to_ast(buf, module_name):
-    """ Import content from buf and return a Python AST."""
-    return hy_compile(import_buffer_to_hst(buf), module_name)
-
-
-def import_file_to_ast(fpath, module_name):
-    """Import content from fpath and return a Python AST."""
-    return hy_compile(import_file_to_hst(fpath), module_name)
-
-
-def import_file_to_module(module_name, fpath, loader=None):
-    """Import Hy source from fpath and put it into a Python module.
-
-    If there's an up-to-date byte-compiled version of this module, load that
-    instead. Otherwise, byte-compile the module once we're done loading it, if
-    we can.
-
-    Return the module."""
-
-    module = None
-
-    bytecode_path = get_bytecode_path(fpath)
-    try:
-        source_mtime = int(os.stat(fpath).st_mtime)
-        with open(bytecode_path, 'rb') as bc_f:
-            # The first 4 bytes are the magic number for the version of Python
-            # that compiled this bytecode.
-            bytecode_magic = bc_f.read(4)
-            # Python 3.7 introduced a new flags entry in the header structure.
-            if PY37:
-                bc_f.read(4)
-            # The next 4 bytes, interpreted as a little-endian 32-bit integer,
-            # are the mtime of the corresponding source file.
-            bytecode_mtime, = struct.unpack('<i', bc_f.read(4))
-    except (IOError, OSError):
-        pass
-    else:
-        if bytecode_magic == MAGIC and bytecode_mtime >= source_mtime:
-            # It's a cache hit. Load the byte-compiled version.
-            if PY3:
-                # As of Python 3.6, imp.load_compiled still exists, but it's
-                # deprecated. So let's use SourcelessFileLoader instead.
-                from importlib.machinery import SourcelessFileLoader
-                module = (SourcelessFileLoader(module_name, bytecode_path).
-                          load_module(module_name))
-            else:
-                module = imp.load_compiled(module_name, bytecode_path)
-
-    if not module:
-        # It's a cache miss, so load from source.
-        sys.modules[module_name] = None
-        try:
-            _ast = import_file_to_ast(fpath, module_name)
-            module = imp.new_module(module_name)
-            module.__file__ = os.path.normpath(fpath)
-            code = ast_compile(_ast, fpath, "exec")
-            if not os.environ.get('PYTHONDONTWRITEBYTECODE'):
-                try:
-                    write_code_as_pyc(fpath, code)
-                except (IOError, OSError):
-                    # We failed to save the bytecode, probably because of a
-                    # permissions issue. The user only asked to import the
-                    # file, so don't bug them about it.
-                    pass
-            eval(code, module.__dict__)
-        except (HyTypeError, LexException) as e:
-            if e.source is None:
-                with open(fpath, 'rt') as fp:
-                    e.source = fp.read()
-                e.filename = fpath
-            raise
-        except Exception:
-            sys.modules.pop(module_name, None)
-            raise
-        sys.modules[module_name] = module
-        module.__name__ = module_name
-
-    module.__file__ = os.path.normpath(fpath)
-    if loader:
-        module.__loader__ = loader
-    if is_package(module_name):
-        module.__path__ = []
-        module.__package__ = module_name
-    else:
-        module.__package__ = module_name.rpartition('.')[0]
-
-    return module
-
-
-def import_buffer_to_module(module_name, buf):
-    try:
-        _ast = import_buffer_to_ast(buf, module_name)
-        mod = imp.new_module(module_name)
-        eval(ast_compile(_ast, "", "exec"), mod.__dict__)
-    except (HyTypeError, LexException) as e:
-        if e.source is None:
-            e.source = buf
-            e.filename = '<stdin>'
-        raise
-    return mod
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    source = re.sub(r'\A#!.*', '', source)
+    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
 
 
 def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
-    """``eval`` evaluates a quoted expression and returns the value. The optional
-    second and third arguments specify the dictionary of globals to use and the
-    module name. The globals dictionary defaults to ``(local)`` and the module
-    name defaults to the name of the current module.
+    """Evaluates a quoted expression and returns the value.
+
+    The optional second and third arguments specify the dictionary of globals
+    to use and the module name. The globals dictionary defaults to ``(local)``
+    and the module name defaults to the name of the current module.
+
+    Examples
+    --------
 
        => (eval '(print "Hello World"))
        "Hello World"
@@ -164,7 +81,31 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     form first:
 
        => (eval (read-str "(+ 1 1)"))
-       2"""
+       2
+
+    Parameters
+    ----------
+    hytree: a Hy expression tree
+        Source code to parse.
+
+    namespace: dict, optional
+        Namespace in which to evaluate the Hy tree.  Defaults to the calling
+        frame.
+
+    module_name: str, optional
+        Name of the module to which the Hy tree is assigned.  Defaults to
+        the calling frame's module, if any, and '__eval__' otherwise.
+
+    ast_callback: callable, optional
+        A callback that is passed the Hy compiled tree and resulting
+        expression object, in that order, after compilation but before
+        evaluation.
+
+    Returns
+    -------
+    out : Result of evaluating the Hy compiled tree.
+
+    """
     if namespace is None:
         frame = inspect.stack()[1][0]
         namespace = inspect.getargvalues(frame).locals
@@ -199,89 +140,427 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     return eval(ast_compile(expr, "<eval>", "eval"), namespace)
 
 
-def write_hy_as_pyc(fname):
-    _ast = import_file_to_ast(fname,
-                              os.path.basename(os.path.splitext(fname)[0]))
-    code = ast_compile(_ast, fname, "exec")
-    write_code_as_pyc(fname, code)
+def cache_from_source(source_path):
+    """Get the cached bytecode file name for a given source file name.
 
+    This function's name is set to mirror Python 3.x's
+    `importlib.util.cache_from_source`, which is also used when available.
 
-def write_code_as_pyc(fname, code):
-    st = os.stat(fname)
-    timestamp = long_type(st.st_mtime)
+    Parameters
+    ----------
+    source_path : str
+        Path of the source file
 
-    cfile = get_bytecode_path(fname)
-    try:
-        os.makedirs(os.path.dirname(cfile))
-    except (IOError, OSError):
-        pass
-
-    with builtins.open(cfile, 'wb') as fc:
-        fc.write(MAGIC)
-        if PY37:
-            # With PEP 552, the header structure has a new flags field
-            # that we need to fill in. All zeros preserve the legacy
-            # behaviour, but should we implement reproducible builds,
-            # this is where we'd add the information.
-            wr_long(fc, 0)
-        wr_long(fc, timestamp)
-        if PY3:
-            wr_long(fc, st.st_size)
-        marshal.dump(code, fc)
-
-
-class MetaLoader(object):
-    def __init__(self, path):
-        self.path = path
-
-    def load_module(self, fullname):
-        if fullname in sys.modules:
-            return sys.modules[fullname]
-
-        if not self.path:
-            return
-
-        return import_file_to_module(fullname, self.path, self)
-
-
-class MetaImporter(object):
-    def find_on_path(self, fullname):
-        fls = ["%s/__init__.hy", "%s.hy"]
-        dirpath = "/".join(fullname.split("."))
-
-        for pth in sys.path:
-            pth = os.path.abspath(pth)
-            for fp in fls:
-                composed_path = fp % ("%s/%s" % (pth, dirpath))
-                if os.path.exists(composed_path):
-                    return composed_path
-
-    def find_module(self, fullname, path=None):
-        path = self.find_on_path(fullname)
-        if path:
-            return MetaLoader(path)
-
-
-sys.meta_path.insert(0, MetaImporter())
-sys.path.insert(0, "")
-
-
-def is_package(module_name):
-    mpath = os.path.join(*module_name.split("."))
-    for path in map(os.path.abspath, sys.path):
-        if os.path.exists(os.path.join(path, mpath, "__init__.hy")):
-            return True
-    return False
-
-
-def get_bytecode_path(source_path):
+    Returns
+    -------
+    out : str
+        Path of the corresponding bytecode file that may--or may
+        not--actually exist.
+    """
     if PY3:
-        import importlib.util
         return importlib.util.cache_from_source(source_path)
-    elif hasattr(imp, "cache_from_source"):
-        return imp.cache_from_source(source_path)
     else:
         # If source_path has a file extension, replace it with ".pyc".
         # Otherwise, just append ".pyc".
         d, f = os.path.split(source_path)
         return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
+
+
+def _get_code_from_file(run_name, fname=None):
+    """A patch of `runpy._get_code_from_file` that will also compile Hy
+    code.
+
+    This version will read and cache bytecode for Hy files.  It operates
+    normally otherwise.
+    """
+    if fname is None and run_name is not None:
+        fname = run_name
+
+    if fname.endswith('.hy'):
+        full_fname = os.path.abspath(fname)
+        fname_path, fname_file = os.path.split(full_fname)
+        modname = os.path.splitext(fname_file)[0]
+        sys.path.insert(0, fname_path)
+        try:
+            loader = pkgutil.get_loader(modname)
+            code = loader.get_code(modname)
+        finally:
+            sys.path.pop(0)
+    else:
+        with open(fname, "rb") as f:
+            code = pkgutil.read_code(f)
+        if code is None:
+            with open(fname, "rb") as f:
+                source = f.read().decode('utf-8')
+            code = compile(source, fname, 'exec')
+
+    return (code, fname) if PY3 else code
+
+
+_runpy_get_code_from_file = runpy._get_code_from_file
+runpy._get_code_from_file = _get_code_from_file
+
+if PY3:
+    importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy')
+    _py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code
+
+    def _hy_source_to_code(self, data, path, _optimize=-1):
+        if os.path.isfile(path) and path.endswith('.hy'):
+            source = data.decode("utf-8")
+            try:
+                hy_tree = hy_parse(source)
+                data = hy_compile(hy_tree, self.name)
+            except (HyTypeError, LexException) as e:
+                if e.source is None:
+                    e.source = source
+                    e.filename = path
+                raise
+
+        return _py_source_to_code(self, data, path, _optimize=_optimize)
+
+    importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code
+
+    #  This is actually needed; otherwise, pre-created finders assigned to the
+    #  current dir (i.e. `''`) in `sys.path` will not catch absolute imports of
+    #  directory-local modules!
+    sys.path_importer_cache.clear()
+
+    # Do this one just in case?
+    importlib.invalidate_caches()
+
+    # XXX: These and the 2.7 counterparts below aren't truly cross-compliant.
+    # They're useful for testing, though.
+    HyImporter = importlib.machinery.FileFinder
+    HyLoader = importlib.machinery.SourceFileLoader
+
+else:
+    import imp
+    import py_compile
+    import marshal
+    import struct
+    import traceback
+
+    from pkgutil import ImpImporter, ImpLoader
+
+    # 100x better!
+    HY_SOURCE = imp.PY_SOURCE * 100
+
+    class HyLoader(ImpLoader, object):
+        def __init__(self, fullname, filename, fileobj=None, etc=None):
+            """This constructor is designed for some compatibility with
+            SourceFileLoader."""
+            if etc is None and filename is not None:
+                if filename.endswith('.hy'):
+                    etc = ('.hy', 'U', HY_SOURCE)
+                if fileobj is None:
+                    fileobj = open(filename, 'rU')
+
+            super(HyLoader, self).__init__(fullname, fileobj, filename, etc)
+
+        def exec_module(self, module, fullname=None):
+            fullname = self._fix_name(fullname)
+            ast = self.get_code(fullname)
+            eval(ast, module.__dict__)
+
+        def load_module(self, fullname=None):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            fullname = self._fix_name(fullname)
+            mod_type = self.etc[2]
+            mod = None
+            pkg_path = os.path.join(self.filename, '__init__.hy')
+            if mod_type == HY_SOURCE or (
+                    mod_type == imp.PKG_DIRECTORY and
+                    os.path.isfile(pkg_path)):
+
+                if fullname in sys.modules:
+                    mod = sys.modules[fullname]
+                else:
+                    mod = sys.modules.setdefault(
+                        fullname, imp.new_module(fullname))
+
+                # TODO: Should we set these only when not in `sys.modules`?
+                if mod_type == imp.PKG_DIRECTORY:
+                    mod.__file__ = pkg_path
+                    mod.__path__ = [self.filename]
+                    mod.__package__ = fullname
+                else:
+                    # mod.__path__ = self.filename
+                    mod.__file__ = self.get_filename(fullname)
+                    mod.__package__ = '.'.join(fullname.split('.')[:-1])
+
+                # TODO: Set `mod.__doc__`.
+                mod.__name__ = fullname
+
+                self.exec_module(mod, fullname=fullname)
+
+            if mod is None:
+                self._reopen()
+                try:
+                    mod = imp.load_module(fullname, self.file, self.filename,
+                                          self.etc)
+                finally:
+                    if self.file:
+                        self.file.close()
+
+            mod.__loader__ = self
+            return mod
+
+        def _reopen(self):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            super(HyLoader, self)._reopen()
+
+            # Add the Hy case...
+            if self.file and self.file.closed:
+                mod_type = self.etc[2]
+                if mod_type == HY_SOURCE:
+                    self.file = io.open(self.filename, 'rU', encoding='utf-8')
+
+        def byte_compile_hy(self, fullname=None):
+            fullname = self._fix_name(fullname)
+            if fullname is None:
+                fullname = self.fullname
+            try:
+                hy_source = self.get_source(fullname)
+                hy_tree = hy_parse(hy_source)
+                ast = hy_compile(hy_tree, fullname)
+                code = compile(ast, self.filename, 'exec',
+                               hy_ast_compile_flags)
+            except (HyTypeError, LexException) as e:
+                if e.source is None:
+                    e.source = hy_source
+                    e.filename = self.filename
+                raise
+
+            if not sys.dont_write_bytecode:
+                try:
+                    hyc_compile(code)
+                except IOError:
+                    pass
+            return code
+
+        def get_code(self, fullname=None):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            fullname = self._fix_name(fullname)
+            mod_type = self.etc[2]
+            if mod_type == HY_SOURCE:
+                # Looks like we have to manually check for--and update--
+                # the bytecode.
+                t_py = long(os.stat(self.filename).st_mtime)
+                pyc_file = cache_from_source(self.filename)
+                if os.path.isfile(pyc_file):
+                    t_pyc = long(os.stat(pyc_file).st_mtime)
+
+                    if t_pyc is not None and t_pyc >= t_py:
+                        with open(pyc_file, 'rb') as f:
+                            if f.read(4) == imp.get_magic():
+                                t = struct.unpack('<I', f.read(4))[0]
+                                if t == t_py:
+                                    self.code = marshal.load(f)
+
+                if self.code is None:
+                    # There's no existing bytecode, or bytecode timestamp
+                    # is older than the source file's.
+                    self.code = self.byte_compile_hy(fullname)
+
+            if self.code is None:
+              super(HyLoader, self).get_code(fullname=fullname)
+
+            return self.code
+
+        def get_source(self, fullname=None):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            fullname = self._fix_name(fullname)
+            mod_type = self.etc[2]
+            if self.source is None and mod_type == HY_SOURCE:
+                self._reopen()
+                try:
+                    self.source = self.file.read()
+                finally:
+                    self.file.close()
+
+            if self.source is None:
+                super(HyLoader, self).get_source(fullname=fullname)
+
+            return self.source
+
+        def get_filename(self, fullname=None):
+            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
+            source"""
+            fullname = self._fix_name(fullname)
+            if self.etc[2] == HY_SOURCE:
+                return self.filename
+
+            if self.filename is None:
+                filename = super(HyLoader, self).get_filename(fullname=fullname)
+
+            return filename
+
+        def _get_delegate(self):
+            return HyImporter(self.filename).find_module('__init__')
+
+    class HyImporter(ImpImporter, object):
+
+        def __init__(self, path=None):
+            # We need to be strict about the types of files this importer will
+            # handle.  To start, if the path is not the current directory in
+            # (represented by '' in `sys.path`), then it must be a supported
+            # file type or a directory.  If it isn't, this importer is not
+            # suitable: throw an exception.
+
+            if path == '' or os.path.isdir(path) or (
+                    os.path.isfile(path) and path.endswith('.hy')):
+                self.path = path
+            else:
+                raise ImportError('Invalid path: {}'.format(path))
+
+        def find_loader(self, fullname):
+            return self.find_module(fullname, path=None)
+
+        def find_module(self, fullname, path=None):
+
+            subname = fullname.split(".")[-1]
+
+            if subname != fullname and self.path is None:
+                return None
+
+            if self.path is None:
+                path = None
+            else:
+                path = [os.path.realpath(self.path)]
+
+            fileobj, file_path, etc = None, None, None
+
+            # The following are excerpts from the later pure Python
+            # implementations of the `imp` module (e.g. in Python 3.6).
+            if path is None:
+                path = sys.path
+
+            for entry in path:
+                if (os.path.isfile(entry) and subname == '__main__' and
+                    entry.endswith('.hy')):
+                    file_path = entry
+                    fileobj = io.open(file_path, 'rU', encoding='utf-8')
+                    etc = ('.hy', 'U', HY_SOURCE)
+                    break
+                else:
+                    file_path = os.path.join(entry, subname)
+                    path_init = os.path.join(file_path, '__init__.hy')
+                    if os.path.isfile(path_init):
+                        fileobj = None
+                        etc = ('', '', imp.PKG_DIRECTORY)
+                        break
+
+                    file_path = file_path + '.hy'
+                    if os.path.isfile(file_path):
+                        fileobj = io.open(file_path, 'rU', encoding='utf-8')
+                        etc = ('.hy', 'U', HY_SOURCE)
+                        break
+            else:
+                try:
+                    fileobj, file_path, etc = imp.find_module(subname, path)
+                except (ImportError, IOError):
+                    return None
+
+
+            return HyLoader(fullname, file_path, fileobj, etc)
+
+    sys.path_hooks.append(HyImporter)
+    sys.path_importer_cache.clear()
+
+    _py_compile_compile = py_compile.compile
+
+    def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False):
+        """Write a Hy file, or code object, to pyc.
+
+        This is a patched version of Python 2.7's `py_compile.compile`.
+
+        Also, it tries its best to write the bytecode file atomically.
+
+        Parameters
+        ----------
+        file_or_code : str or instance of `types.CodeType`
+            A filename for a Hy or Python source file or its corresponding code
+            object.
+        cfile : str, optional
+            The filename to use for the bytecode file.  If `None`, use the
+            standard bytecode filename determined by `cache_from_source`.
+        dfile : str, optional
+            The filename to use for compile-time errors.
+        doraise : bool, default False
+            If `True` raise compilation exceptions; otherwise, ignore them.
+
+        Returns
+        -------
+        out : str
+            The resulting bytecode file name.  Python 3.x returns this, but
+            Python 2.7 doesn't; this function does for convenience.
+        """
+
+        if isinstance(file_or_code, types.CodeType):
+            codeobject = file_or_code
+            filename = codeobject.co_filename
+        else:
+            filename = file_or_code
+
+            with open(filename, 'rb') as f:
+                source_str = f.read().decode('utf-8')
+
+            try:
+                flags = None
+                if filename.endswith('.hy'):
+                    hy_tree = hy_parse(source_str)
+                    source = hy_compile(hy_tree, '<hyc_compile>')
+                    flags = hy_ast_compile_flags
+
+                codeobject = compile(source, dfile or filename, 'exec', flags)
+            except Exception as err:
+                if isinstance(err, (HyTypeError, LexException)) and err.source is None:
+                    err.source = source_str
+                    err.filename = filename
+
+                py_exc = py_compile.PyCompileError(err.__class__, err,
+                                                   dfile or filename)
+                if doraise:
+                    raise py_exc
+                else:
+                    traceback.print_exc()
+                    return
+
+        timestamp = long(os.stat(filename).st_mtime)
+
+        if cfile is None:
+            cfile = cache_from_source(filename)
+
+        f = tempfile.NamedTemporaryFile('wb', dir=os.path.split(cfile)[0],
+                                        delete=False)
+        try:
+            f.write('\0\0\0\0')
+            f.write(struct.pack('<I', timestamp))
+            f.write(marshal.dumps(codeobject))
+            f.flush()
+            f.seek(0, 0)
+            f.write(imp.get_magic())
+
+            # Make sure it's written to disk.
+            f.flush()
+            os.fsync(f.fileno())
+            f.close()
+
+            # Rename won't replace an existing dest on Windows.
+            if os.name == 'nt' and os.path.isfile(cfile):
+                os.unlink(cfile)
+
+            os.rename(f.name, cfile)
+        except OSError:
+            try:
+                os.unlink(f.name)
+            except OSError:
+                pass
+
+        return cfile
+
+    py_compile.compile = hyc_compile
diff --git a/hy/macros.py b/hy/macros.py
index c5ab844..a86f043 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -1,6 +1,10 @@
 # Copyright 2018 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
+import pkgutil
+import importlib
+
+from collections import defaultdict
 
 from hy._compat import PY3
 import hy.inspect
@@ -10,8 +14,6 @@ from hy._compat import str_type
 
 from hy.errors import HyTypeError, HyMacroExpansionError
 
-from collections import defaultdict
-
 CORE_MACROS = [
     "hy.core.bootstrap",
 ]
@@ -120,20 +122,14 @@ def load_macros(module_name):
     Other modules get the macros from CORE_MACROS and EXTRA_MACROS.
 
     """
-
-    def _import(module, module_name=module_name):
-        "__import__ a module, avoiding recursions"
-        if module != module_name:
-            __import__(module)
-
     for module in CORE_MACROS:
-        _import(module)
+        importlib.import_module(module)
 
     if module_name.startswith("hy.core"):
         return
 
     for module in EXTRA_MACROS:
-        _import(module)
+        importlib.import_module(module)
 
 
 def make_empty_fn_copy(fn):
@@ -159,8 +155,8 @@ def make_empty_fn_copy(fn):
 def macroexpand(tree, compiler, once=False):
     """Expand the toplevel macros for the `tree`.
 
-    Load the macros from the given `module_name`, then expand the (top-level)
-    macros in `tree` until we no longer can.
+    Load the macros from the given `compiler.module_name`, then expand the
+    (top-level) macros in `tree` until we no longer can.
 
     """
     load_macros(compiler.module_name)
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 322f78c..3a1e52a 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -7,8 +7,7 @@ from __future__ import unicode_literals
 
 from hy import HyString
 from hy.models import HyObject
-from hy.compiler import hy_compile
-from hy.importer import hy_eval, import_buffer_to_hst
+from hy.importer import hy_compile, hy_eval, hy_parse
 from hy.errors import HyCompileError, HyTypeError
 from hy.lex.exceptions import LexException
 from hy._compat import PY3
@@ -16,6 +15,7 @@ from hy._compat import PY3
 import ast
 import pytest
 
+
 def _ast_spotcheck(arg, root, secondary):
     if "." in arg:
         local, full = arg.split(".", 1)
@@ -26,16 +26,16 @@ def _ast_spotcheck(arg, root, secondary):
 
 
 def can_compile(expr):
-    return hy_compile(import_buffer_to_hst(expr), "__main__")
+    return hy_compile(hy_parse(expr), "__main__")
 
 
 def can_eval(expr):
-    return hy_eval(import_buffer_to_hst(expr))
+    return hy_eval(hy_parse(expr))
 
 
 def cant_compile(expr):
     try:
-        hy_compile(import_buffer_to_hst(expr), "__main__")
+        hy_compile(hy_parse(expr), "__main__")
         assert False
     except HyTypeError as e:
         # Anything that can't be compiled should raise a user friendly
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index 8aca7c9..c41f86b 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -2,35 +2,43 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
-import hy
-from hy.importer import (import_file_to_module, import_buffer_to_ast,
-                         MetaLoader, get_bytecode_path)
-from hy.errors import HyTypeError
 import os
 import ast
 import tempfile
+import importlib
+import runpy
+
 from fractions import Fraction
+
 import pytest
 
+import hy
+from hy.errors import HyTypeError
+from hy.compiler import hy_compile
+from hy.importer import hy_parse, HyLoader, cache_from_source
+
 
 def test_basics():
     "Make sure the basics of the importer work"
-    import_file_to_module("basic",
-                          "tests/resources/importer/basic.hy")
+
+    basic_namespace = runpy.run_path("tests/resources/importer/basic.hy",
+                                     run_name='__main__')
+    assert 'square' in basic_namespace
 
 
 def test_stringer():
-    _ast = import_buffer_to_ast("(defn square [x] (* x x))", '')
+    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '')
+
     assert type(_ast.body[0]) == ast.FunctionDef
 
 
 def test_imports():
     path = os.getcwd() + "/tests/resources/importer/a.hy"
-    testLoader = MetaLoader(path)
+    testLoader = HyLoader("tests.resources.importer.a", path)
 
     def _import_test():
         try:
-            return testLoader.load_module("tests.resources.importer.a")
+            return testLoader.load_module()
         except:
             return "Error"
 
@@ -43,7 +51,7 @@ def test_import_error_reporting():
 
     def _import_error_test():
         try:
-            import_buffer_to_ast("(import \"sys\")", '')
+            _ = hy_compile(hy_parse("(import \"sys\")"), '')
         except HyTypeError:
             return "Error reported"
 
@@ -56,19 +64,23 @@ def test_import_error_reporting():
 def test_import_autocompiles():
     "Test that (import) byte-compiles the module."
 
-    f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False)
-    f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
-    f.close()
+    with tempfile.NamedTemporaryFile(suffix='.hy', delete=True) as f:
+        f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
+        f.flush()
 
-    try:
-        os.remove(get_bytecode_path(f.name))
-    except (IOError, OSError):
-        pass
-    import_file_to_module("mymodule", f.name)
-    assert os.path.exists(get_bytecode_path(f.name))
+        pyc_path = cache_from_source(f.name)
 
-    os.remove(f.name)
-    os.remove(get_bytecode_path(f.name))
+        try:
+            os.remove(pyc_path)
+        except (IOError, OSError):
+            pass
+
+        test_loader = HyLoader("mymodule", f.name).load_module()
+
+        assert hasattr(test_loader, 'pyctest')
+        assert os.path.exists(pyc_path)
+
+        os.remove(pyc_path)
 
 
 def test_eval():
diff --git a/tests/importer/test_pyc.py b/tests/importer/test_pyc.py
index d85b630..ad37617 100644
--- a/tests/importer/test_pyc.py
+++ b/tests/importer/test_pyc.py
@@ -5,20 +5,21 @@
 import os
 import imp
 import tempfile
-from hy.importer import write_hy_as_pyc, get_bytecode_path
+
+import py_compile
 
 
 def test_pyc():
     """Test pyc compilation."""
-    f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False)
-    f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
-    f.close()
+    with tempfile.NamedTemporaryFile(suffix='.hy') as f:
+        f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
+        f.flush()
 
-    write_hy_as_pyc(f.name)
-    os.remove(f.name)
+        cfile = py_compile.compile(f.name)
 
-    cfile = get_bytecode_path(f.name)
-    mod = imp.load_compiled('pyc', cfile)
-    os.remove(cfile)
+        assert os.path.exists(cfile)
 
-    assert mod.pyctest('Foo') == 'XFooY'
+        mod = imp.load_compiled('pyc', cfile)
+        os.remove(cfile)
+
+        assert mod.pyctest('Foo') == 'XFooY'
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index e58c911..ef6ae26 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -147,7 +147,7 @@
 (defn test-gensym-in-macros []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (setv g (gensym))
       `(do
@@ -160,8 +160,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   ;; and make sure there is something new that starts with _;G|
@@ -173,7 +173,7 @@
 (defn test-with-gensym []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (with-gensyms [a]
         `(do
@@ -186,8 +186,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;a|") s1))
@@ -197,7 +197,7 @@
 (defn test-defmacro/g! []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro/g! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
@@ -209,8 +209,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;res|") s1))
@@ -220,13 +220,13 @@
   ;; defmacro/g! didn't like numbers initially because they
   ;; don't have a startswith method and blew up during expansion
   (setv macro2 "(defmacro/g! two-point-zero [] `(+ (float 1) 1.0))")
-  (assert (import_buffer_to_ast macro2 "foo")))
+  (assert (hy-compile (hy-parse macro2) "foo")))
 
 (defn test-defmacro! []
   ;; defmacro! must do everything defmacro/g! can
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [import_buffer_to_ast]])
+  (import [hy.importer [hy-parse hy-compile]])
   (setv macro1 "(defmacro! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
@@ -238,8 +238,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (import_buffer_to_ast macro1 "foo"))
-  (setv _ast2 (import_buffer_to_ast macro1 "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;res|") s1))
@@ -249,7 +249,7 @@
   ;; defmacro/g! didn't like numbers initially because they
   ;; don't have a startswith method and blew up during expansion
   (setv macro2 "(defmacro! two-point-zero [] `(+ (float 1) 1.0))")
-  (assert (import_buffer_to_ast macro2 "foo"))
+  (assert (hy-compile (hy-parse macro2) "foo"))
 
   (defmacro! foo! [o!foo] `(do ~g!foo ~g!foo))
   ;; test that o! becomes g!
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 1a32e01..6ccaea1 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -9,10 +9,11 @@ import re
 import shlex
 import subprocess
 
+from hy.importer import cache_from_source
+
 import pytest
 
 from hy._compat import builtins
-from hy.importer import get_bytecode_path
 
 
 hy_dir = os.environ.get('HY_DIR', '')
@@ -231,8 +232,11 @@ def test_bin_hy_file_with_args():
 
 
 def test_bin_hyc():
-    _, err = run_cmd("hyc", expect=2)
-    assert "usage" in err
+    _, err = run_cmd("hyc", expect=0)
+    assert err == ''
+
+    _, err = run_cmd("hyc -", expect=0)
+    assert err == ''
 
     output, _ = run_cmd("hyc -h")
     assert "usage" in output
@@ -240,12 +244,12 @@ def test_bin_hyc():
     path = "tests/resources/argparse_ex.hy"
     output, _ = run_cmd("hyc " + path)
     assert "Compiling" in output
-    assert os.path.exists(get_bytecode_path(path))
-    rm(get_bytecode_path(path))
+    assert os.path.exists(cache_from_source(path))
+    rm(cache_from_source(path))
 
 
 def test_bin_hyc_missing_file():
-    _, err = run_cmd("hyc foobarbaz", expect=2)
+    _, err = run_cmd("hyc foobarbaz", expect=1)
     assert "[Errno 2]" in err
 
 
@@ -284,19 +288,19 @@ def test_bin_hy_no_main():
 @pytest.mark.parametrize('scenario', [
     "normal", "prevent_by_force", "prevent_by_env"])
 @pytest.mark.parametrize('cmd_fmt', [
-    'hy {fpath}', 'hy -m {modname}', "hy -c '(import {modname})'"])
+    'hy -m {modname}', "hy -c '(import {modname})'"])
 def test_bin_hy_byte_compile(scenario, cmd_fmt):
 
     modname = "tests.resources.bin.bytecompile"
     fpath = modname.replace(".", "/") + ".hy"
     cmd = cmd_fmt.format(**locals())
 
-    rm(get_bytecode_path(fpath))
+    rm(cache_from_source(fpath))
 
     if scenario == "prevent_by_force":
         # Keep Hy from being able to byte-compile the module by
         # creating a directory at the target location.
-        os.mkdir(get_bytecode_path(fpath))
+        os.mkdir(cache_from_source(fpath))
 
     # Whether or not we can byte-compile the module, we should be able
     # to run it.
@@ -306,10 +310,10 @@ def test_bin_hy_byte_compile(scenario, cmd_fmt):
 
     if scenario == "normal":
         # That should've byte-compiled the module.
-        assert os.path.exists(get_bytecode_path(fpath))
+        assert os.path.exists(cache_from_source(fpath))
     elif scenario == "prevent_by_env":
         # No byte-compiled version should've been created.
-        assert not os.path.exists(get_bytecode_path(fpath))
+        assert not os.path.exists(cache_from_source(fpath))
 
     # When we run the same command again, and we've byte-compiled the
     # module, the byte-compiled version should be run instead of the

From e9e7171b5653e6ea2c7c8952c2960d54089e67ca Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Mon, 20 Aug 2018 10:27:31 -0500
Subject: [PATCH 061/223] Add module reloading tests

---
 tests/importer/test_importer.py | 102 +++++++++++++++++++++++++++++++-
 1 file changed, 101 insertions(+), 1 deletion(-)

diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index c41f86b..d9c3565 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -3,10 +3,12 @@
 # license. See the LICENSE.
 
 import os
+import sys
 import ast
+import imp
 import tempfile
-import importlib
 import runpy
+import importlib
 
 from fractions import Fraction
 
@@ -14,6 +16,7 @@ import pytest
 
 import hy
 from hy.errors import HyTypeError
+from hy.lex import LexException
 from hy.compiler import hy_compile
 from hy.importer import hy_parse, HyLoader, cache_from_source
 
@@ -98,3 +101,100 @@ def test_eval():
         '(if True "this is if true" "this is if false")') == "this is if true"
     assert eval_str('(lfor num (range 100) :if (= (% num 2) 1) (pow num 2))') == [
         pow(num, 2) for num in range(100) if num % 2 == 1]
+
+
+def test_reload():
+    """Copied from CPython's `test_import.py`"""
+
+    def unlink(filename):
+        os.unlink(source)
+        bytecode = cache_from_source(source)
+        if os.path.isfile(bytecode):
+            os.unlink(bytecode)
+
+    TESTFN = 'testfn'
+    source = TESTFN + os.extsep + "hy"
+    with open(source, "w") as f:
+        f.write("(setv a 1)")
+        f.write("(setv b 2)")
+
+    sys.path.insert(0, os.curdir)
+    try:
+        mod = importlib.import_module(TESTFN)
+        assert TESTFN in sys.modules
+        assert mod.a == 1
+        assert mod.b == 2
+
+        # On WinXP, just replacing the .py file wasn't enough to
+        # convince reload() to reparse it.  Maybe the timestamp didn't
+        # move enough.  We force it to get reparsed by removing the
+        # compiled file too.
+        unlink(source)
+
+        # Now damage the module.
+        with open(source, "w") as f:
+            f.write("(setv a 10)")
+            f.write("(setv b (// 20 0))")
+
+        with pytest.raises(ZeroDivisionError):
+            imp.reload(mod)
+
+        # But we still expect the module to be in sys.modules.
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        # We should have replaced a w/ 10, but the old b value should
+        # stick.
+        assert mod.a == 10
+        assert mod.b == 2
+
+        # Now fix the issue and reload the module.
+        unlink(source)
+
+        with open(source, "w") as f:
+            f.write("(setv a 11)")
+            f.write("(setv b (// 20 1))")
+
+        imp.reload(mod)
+
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        assert mod.a == 11
+        assert mod.b == 20
+
+        # Now cause a LexException
+        unlink(source)
+
+        with open(source, "w") as f:
+            f.write("(setv a 11")
+            f.write("(setv b (// 20 1))")
+
+        with pytest.raises(LexException):
+            imp.reload(mod)
+
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        assert mod.a == 11
+        assert mod.b == 20
+
+        # Fix it and retry
+        unlink(source)
+
+        with open(source, "w") as f:
+            f.write("(setv a 12)")
+            f.write("(setv b (// 10 1))")
+
+        imp.reload(mod)
+
+        mod = sys.modules.get(TESTFN)
+        assert mod is not None
+
+        assert mod.a == 12
+        assert mod.b == 10
+
+    finally:
+        del sys.path[0]
+        unlink(source)
+        del sys.modules[TESTFN]

From 4839acadf7bcce342995ebac1c90b33708aae11f Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 22 Aug 2018 01:38:15 -0500
Subject: [PATCH 062/223] Add tests for importing __main__.hy files

Closes hylang/hy#1466.
---
 tests/importer/test_importer.py | 28 +++++++++++++++++++++++++---
 tests/resources/bin/__main__.hy |  2 ++
 tests/test_bin.py               | 12 ++++++++++++
 3 files changed, 39 insertions(+), 3 deletions(-)
 create mode 100644 tests/resources/bin/__main__.hy

diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index d9c3565..db1e2a3 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -24,9 +24,31 @@ from hy.importer import hy_parse, HyLoader, cache_from_source
 def test_basics():
     "Make sure the basics of the importer work"
 
-    basic_namespace = runpy.run_path("tests/resources/importer/basic.hy",
-                                     run_name='__main__')
-    assert 'square' in basic_namespace
+    assert os.path.isfile('tests/resources/__init__.py')
+    resources_mod = importlib.import_module('tests.resources')
+    assert hasattr(resources_mod, 'kwtest')
+
+    assert os.path.isfile('tests/resources/bin/__init__.hy')
+    bin_mod = importlib.import_module('tests.resources.bin')
+    assert hasattr(bin_mod, '_null_fn_for_import_test')
+
+
+def test_runpy():
+    # XXX: `runpy` won't update cached bytecode!  Don't know if that's
+    # intentional or not.
+
+    basic_ns = runpy.run_path('tests/resources/importer/basic.hy')
+    assert 'square' in basic_ns
+
+    main_ns = runpy.run_path('tests/resources/bin')
+    assert main_ns['visited_main'] == 1
+    del main_ns
+
+    main_ns = runpy.run_module('tests.resources.bin')
+    assert main_ns['visited_main'] == 1
+
+    with pytest.raises(IOError):
+        runpy.run_path('tests/resources/foobarbaz.py')
 
 
 def test_stringer():
diff --git a/tests/resources/bin/__main__.hy b/tests/resources/bin/__main__.hy
new file mode 100644
index 0000000..2f4951d
--- /dev/null
+++ b/tests/resources/bin/__main__.hy
@@ -0,0 +1,2 @@
+(print "This is a __main__.hy")
+(setv visited_main True)
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 6ccaea1..b267bb1 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -328,6 +328,18 @@ def test_bin_hy_module_main():
     assert "Hello World" in output
 
 
+def test_bin_hy_module_main_file():
+    output, _ = run_cmd("hy -m tests.resources.bin")
+    assert "This is a __main__.hy" in output
+
+    output, _ = run_cmd("hy -m .tests.resources.bin", expect=1)
+
+
+def test_bin_hy_file_main_file():
+    output, _ = run_cmd("hy tests/resources/bin")
+    assert "This is a __main__.hy" in output
+
+
 def test_bin_hy_module_main_args():
     output, _ = run_cmd("hy -m tests.resources.bin.main test 123")
     assert "test" in output

From c022abc831c7d5703628a8218bfb21746785d040 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 22 Aug 2018 13:20:07 -0500
Subject: [PATCH 063/223] Add Python cmdline bytecode option and set
 sys.executable

Closes hylang/hy#459.
---
 hy/cmdline.py                   | 22 ++++++++++++++--------
 tests/importer/test_importer.py |  2 +-
 tests/test_bin.py               | 25 ++++++++++++++++++-------
 3 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/hy/cmdline.py b/hy/cmdline.py
index f1c7a9e..f78008e 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -262,6 +262,8 @@ def cmdline_handler(scriptname, argv):
                         help="module to run, passed in as a string")
     parser.add_argument("-E", action='store_true',
                         help="ignore PYTHON* environment variables")
+    parser.add_argument("-B", action='store_true',
+                        help="don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x")
     parser.add_argument("-i", dest="icommand",
                         help="program passed in as a string, then stay in REPL")
     parser.add_argument("--spy", action="store_true",
@@ -278,13 +280,17 @@ def cmdline_handler(scriptname, argv):
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
 
-    # stash the hy executable in case we need it later
-    # mimics Python sys.executable
+    # Get the path of the Hy cmdline executable and swap it with
+    # `sys.executable` (saving the original, just in case).
+    # XXX: The `__main__` module will also have `__file__` set to the
+    # entry-point script.  Currently, I don't see an immediate problem, but
+    # that's not how the Python cmdline works.
     hy.executable = argv[0]
+    hy.sys_executable = sys.executable
+    sys.executable = hy.executable
 
-    # need to split the args if using "-m"
-    # all args after the MOD are sent to the module
-    # in sys.argv
+    # Need to split the args.  If using "-m" all args after the MOD are sent to
+    # the module in sys.argv.
     module_args = []
     if "-m" in argv:
         mloc = argv.index("-m")
@@ -298,13 +304,13 @@ def cmdline_handler(scriptname, argv):
         global SIMPLE_TRACEBACKS
         SIMPLE_TRACEBACKS = False
 
-    # reset sys.argv like Python
-    # sys.argv = [sys.argv[0]] + options.args + module_args
-
     if options.E:
         # User did "hy -E ..."
         _remove_python_envs()
 
+    if options.B:
+        sys.dont_write_bytecode = True
+
     if options.command:
         # User did "hy -c ..."
         return run_command(options.command)
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index db1e2a3..8b0e720 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -84,7 +84,7 @@ def test_import_error_reporting():
     assert _import_error_test() is not None
 
 
-@pytest.mark.skipif(os.environ.get('PYTHONDONTWRITEBYTECODE'),
+@pytest.mark.skipif(sys.dont_write_bytecode,
                     reason="Bytecode generation is suppressed")
 def test_import_autocompiles():
     "Test that (import) byte-compiles the module."
diff --git a/tests/test_bin.py b/tests/test_bin.py
index b267bb1..8827fd6 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -6,6 +6,7 @@
 
 import os
 import re
+import sys
 import shlex
 import subprocess
 
@@ -285,15 +286,20 @@ def test_bin_hy_no_main():
     assert "This Should Still Work" in output
 
 
-@pytest.mark.parametrize('scenario', [
-    "normal", "prevent_by_force", "prevent_by_env"])
-@pytest.mark.parametrize('cmd_fmt', [
-    'hy -m {modname}', "hy -c '(import {modname})'"])
+@pytest.mark.parametrize('scenario', ["normal", "prevent_by_force",
+                                      "prevent_by_env", "prevent_by_option"])
+@pytest.mark.parametrize('cmd_fmt', [['hy', '{fpath}'],
+                                     ['hy', '-m', '{modname}'],
+                                     ['hy', '-c', "'(import {modname})'"]])
 def test_bin_hy_byte_compile(scenario, cmd_fmt):
 
     modname = "tests.resources.bin.bytecompile"
     fpath = modname.replace(".", "/") + ".hy"
-    cmd = cmd_fmt.format(**locals())
+
+    if scenario == 'prevent_by_option':
+        cmd_fmt.insert(1, '-B')
+
+    cmd = ' '.join(cmd_fmt).format(**locals())
 
     rm(cache_from_source(fpath))
 
@@ -304,14 +310,14 @@ def test_bin_hy_byte_compile(scenario, cmd_fmt):
 
     # Whether or not we can byte-compile the module, we should be able
     # to run it.
-    output, _ = run_cmd(cmd, dontwritebytecode=scenario == "prevent_by_env")
+    output, _ = run_cmd(cmd, dontwritebytecode=(scenario == "prevent_by_env"))
     assert "Hello from macro" in output
     assert "The macro returned: boink" in output
 
     if scenario == "normal":
         # That should've byte-compiled the module.
         assert os.path.exists(cache_from_source(fpath))
-    elif scenario == "prevent_by_env":
+    elif scenario == "prevent_by_env" or scenario == "prevent_by_option":
         # No byte-compiled version should've been created.
         assert not os.path.exists(cache_from_source(fpath))
 
@@ -353,3 +359,8 @@ def test_bin_hy_module_main_exitvalue():
 def test_bin_hy_module_no_main():
     output, _ = run_cmd("hy -m tests.resources.bin.nomain")
     assert "This Should Still Work" in output
+
+
+def test_bin_hy_sys_executable():
+    output, _ = run_cmd("hy -c '(do (import sys) (print sys.executable))'")
+    assert output.strip().endswith('/hy')

From cbaba4a10ae702e7ebdfe697c74ec57b318a41aa Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 22 Aug 2018 15:21:17 -0500
Subject: [PATCH 064/223] Use Python cmdline file-relative sys.path

Closes hylang/hy#1457.
---
 hy/cmdline.py                      | 21 +++++++++++++++++++--
 tests/resources/relative_import.hy |  3 +++
 tests/test_bin.py                  | 18 ++++++++++++++++++
 3 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100644 tests/resources/relative_import.hy

diff --git a/hy/cmdline.py b/hy/cmdline.py
index f78008e..08f6c0b 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -229,13 +229,21 @@ def run_repl(hr=None, **kwargs):
 
 
 def run_icommand(source, **kwargs):
-    hr = HyREPL(**kwargs)
     if os.path.exists(source):
+        # Emulate Python cmdline behavior by setting `sys.path` relative
+        # to the executed file's location.
+        if sys.path[0] == '':
+            sys.path[0] = os.path.realpath(os.path.split(source)[0])
+        else:
+            sys.path.insert(0, os.path.split(source)[0])
+
         with io.open(source, "r", encoding='utf-8') as f:
             source = f.read()
         filename = source
     else:
         filename = '<input>'
+
+    hr = HyREPL(**kwargs)
     hr.runsource(source, filename=filename, symbol='single')
     return run_repl(hr)
 
@@ -333,9 +341,18 @@ def cmdline_handler(scriptname, argv):
 
         else:
             # User did "hy <filename>"
+            filename = options.args[0]
+
+            # Emulate Python cmdline behavior by setting `sys.path` relative
+            # to the executed file's location.
+            if sys.path[0] == '':
+                sys.path[0] = os.path.realpath(os.path.split(filename)[0])
+            else:
+                sys.path.insert(0, os.path.split(filename)[0])
+
             try:
                 sys.argv = options.args
-                runpy.run_path(options.args[0], run_name='__main__')
+                runpy.run_path(filename, run_name='__main__')
                 return 0
             except FileNotFoundError as e:
                 print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
diff --git a/tests/resources/relative_import.hy b/tests/resources/relative_import.hy
new file mode 100644
index 0000000..65acc66
--- /dev/null
+++ b/tests/resources/relative_import.hy
@@ -0,0 +1,3 @@
+(import bin.printenv)
+(import sys)
+(print sys.path)
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 8827fd6..e642f53 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -214,6 +214,10 @@ def test_bin_hy_icmd_file():
     output, _ = run_cmd("hy -i resources/icmd_test_file.hy", "(ideas)")
     assert "Hy!" in output
 
+    file_relative_path = os.path.realpath(os.path.split('tests/resources/relative_import.hy')[0])
+
+    output, _ = run_cmd("hy -i tests/resources/relative_import.hy None")
+    assert file_relative_path in output
 
 def test_bin_hy_icmd_and_spy():
     output, _ = run_cmd("hy -i \"(+ [] [])\" --spy", "(+ 1 1)")
@@ -346,6 +350,20 @@ def test_bin_hy_file_main_file():
     assert "This is a __main__.hy" in output
 
 
+def test_bin_hy_file_sys_path():
+    """The test resource `relative_import.hy` will perform an absolute import
+    of a module in its directory: a directory that is not on the `sys.path` of
+    the script executing the module (i.e. `hy`).  We want to make sure that Hy
+    adopts the file's location in `sys.path`, instead of the runner's current
+    dir (e.g. '' in `sys.path`).
+    """
+    file_path, _ = os.path.split('tests/resources/relative_import.hy')
+    file_relative_path = os.path.realpath(file_path)
+
+    output, _ = run_cmd("hy tests/resources/relative_import.hy")
+    assert file_relative_path in output
+
+
 def test_bin_hy_module_main_args():
     output, _ = run_cmd("hy -m tests.resources.bin.main test 123")
     assert "test" in output

From 1da29417fea8c618bfcba1f97de28e7f52a8d831 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 22 Aug 2018 17:32:57 -0500
Subject: [PATCH 065/223] Add test for circular imports

Closes hylang/hy#1134.
---
 tests/importer/test_importer.py      | 12 ++++++++++++
 tests/resources/importer/circular.hy |  5 +++++
 2 files changed, 17 insertions(+)
 create mode 100644 tests/resources/importer/circular.hy

diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index 8b0e720..f10c2a6 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -15,6 +15,7 @@ from fractions import Fraction
 import pytest
 
 import hy
+from hy._compat import bytes_type
 from hy.errors import HyTypeError
 from hy.lex import LexException
 from hy.compiler import hy_compile
@@ -220,3 +221,14 @@ def test_reload():
         del sys.path[0]
         unlink(source)
         del sys.modules[TESTFN]
+
+
+def test_circular():
+    """Test circular imports by creating a temporary file/module that calls a
+    function that imports itself."""
+    sys.path.insert(0, os.path.abspath('tests/resources/importer'))
+    try:
+        mod = runpy.run_module('circular')
+        assert mod['f']() == 1
+    finally:
+        sys.path.pop(0)
diff --git a/tests/resources/importer/circular.hy b/tests/resources/importer/circular.hy
new file mode 100644
index 0000000..4825b5f
--- /dev/null
+++ b/tests/resources/importer/circular.hy
@@ -0,0 +1,5 @@
+(setv a 1)
+(defn f []
+  (import circular)
+  circular.a)
+(print (f))

From bbc66d00425f32ed1669f4ad301cc2fceb02e3ac Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Fri, 24 Aug 2018 18:50:58 -0500
Subject: [PATCH 066/223] Add test for shadowed-basename imports

This test ensures that Hy will load a `.hy` instead of a `.py` counterpart.
---
 tests/importer/test_importer.py          | 21 +++++++++++++++++++++
 tests/resources/importer/foo/__init__.hy |  2 ++
 tests/resources/importer/foo/__init__.py |  2 ++
 tests/resources/importer/foo/some_mod.hy |  2 ++
 tests/resources/importer/foo/some_mod.py |  2 ++
 5 files changed, 29 insertions(+)
 create mode 100644 tests/resources/importer/foo/__init__.hy
 create mode 100644 tests/resources/importer/foo/__init__.py
 create mode 100644 tests/resources/importer/foo/some_mod.hy
 create mode 100644 tests/resources/importer/foo/some_mod.py

diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index f10c2a6..bb92bce 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -232,3 +232,24 @@ def test_circular():
         assert mod['f']() == 1
     finally:
         sys.path.pop(0)
+
+
+def test_shadowed_basename():
+    """Make sure Hy loads `.hy` files instead of their `.py` counterparts (.e.g
+    `__init__.py` and `__init__.hy`).
+    """
+    sys.path.insert(0, os.path.realpath('tests/resources/importer'))
+    try:
+        assert os.path.isfile('tests/resources/importer/foo/__init__.hy')
+        assert os.path.isfile('tests/resources/importer/foo/__init__.py')
+        assert os.path.isfile('tests/resources/importer/foo/some_mod.hy')
+        assert os.path.isfile('tests/resources/importer/foo/some_mod.py')
+
+        foo = importlib.import_module('foo')
+        assert foo.__file__.endswith('foo/__init__.hy')
+        assert foo.ext == 'hy'
+        some_mod = importlib.import_module('foo.some_mod')
+        assert some_mod.__file__.endswith('foo/some_mod.hy')
+        assert some_mod.ext == 'hy'
+    finally:
+        sys.path.pop(0)
diff --git a/tests/resources/importer/foo/__init__.hy b/tests/resources/importer/foo/__init__.hy
new file mode 100644
index 0000000..5909d57
--- /dev/null
+++ b/tests/resources/importer/foo/__init__.hy
@@ -0,0 +1,2 @@
+(print "This is __init__.hy")
+(setv ext "hy")
diff --git a/tests/resources/importer/foo/__init__.py b/tests/resources/importer/foo/__init__.py
new file mode 100644
index 0000000..5ea3615
--- /dev/null
+++ b/tests/resources/importer/foo/__init__.py
@@ -0,0 +1,2 @@
+print('This is __init__.py')
+ext = 'py'
diff --git a/tests/resources/importer/foo/some_mod.hy b/tests/resources/importer/foo/some_mod.hy
new file mode 100644
index 0000000..10db45c
--- /dev/null
+++ b/tests/resources/importer/foo/some_mod.hy
@@ -0,0 +1,2 @@
+(print "This is test_mod.hy")
+(setv ext "hy")
diff --git a/tests/resources/importer/foo/some_mod.py b/tests/resources/importer/foo/some_mod.py
new file mode 100644
index 0000000..d9533b2
--- /dev/null
+++ b/tests/resources/importer/foo/some_mod.py
@@ -0,0 +1,2 @@
+print('This is test_mod.py')
+ext = 'py'

From b12fd33e6f629fd1b71edfd9d8a280cc993f4c17 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Fri, 24 Aug 2018 20:42:48 -0500
Subject: [PATCH 067/223] Update NEWS

---
 NEWS.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 5a226f8..1fe1e41 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -15,6 +15,9 @@ New Features
 
 Bug Fixes
 ------------------------------
+* Fixed module reloading.
+* Fixed circular imports.
+* Fixed `__main__` file execution.
 * Fixed bugs in the handling of unpacking forms in method calls and
   attribute access.
 * Fixed crashes on Windows when calling `hy-repr` on date and time

From 32033b03cefa0d7cd958c5f4aa1c81ad814b63ad Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Fri, 24 Aug 2018 21:56:00 -0500
Subject: [PATCH 068/223] Fix pytest hook so that ignore works consistently

---
 conftest.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/conftest.py b/conftest.py
index 5b1c41d..7d84e11 100644
--- a/conftest.py
+++ b/conftest.py
@@ -12,9 +12,9 @@ _fspath_pyimport = py.path.local.pyimport
 
 
 def pytest_ignore_collect(path, config):
-    return (("py3_only" in path.basename and not PY3)
-            or ("py35_only" in path.basename and not PY35)
-            or ("py36_only" in path.basename and not PY36))
+    return (("py3_only" in path.basename and not PY3) or
+            ("py35_only" in path.basename and not PY35) or
+            ("py36_only" in path.basename and not PY36) or None)
 
 
 def pyimport_patch_mismatch(self, **kwargs):

From 2ea1e8e0170ea4dca593bf182cc545b1e5acb9ea Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sun, 26 Aug 2018 12:10:40 -0500
Subject: [PATCH 069/223] Make Hy a Python-source module type

---
 hy/importer.py | 61 ++++++++++++--------------------------------------
 1 file changed, 14 insertions(+), 47 deletions(-)

diff --git a/hy/importer.py b/hy/importer.py
index 390f373..5cf7435 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -242,18 +242,15 @@ else:
 
     from pkgutil import ImpImporter, ImpLoader
 
-    # 100x better!
-    HY_SOURCE = imp.PY_SOURCE * 100
-
     class HyLoader(ImpLoader, object):
         def __init__(self, fullname, filename, fileobj=None, etc=None):
             """This constructor is designed for some compatibility with
             SourceFileLoader."""
             if etc is None and filename is not None:
                 if filename.endswith('.hy'):
-                    etc = ('.hy', 'U', HY_SOURCE)
-                if fileobj is None:
-                    fileobj = open(filename, 'rU')
+                    etc = ('.hy', 'U', imp.PY_SOURCE)
+                    if fileobj is None:
+                        fileobj = io.open(filename, 'rU', encoding='utf-8')
 
             super(HyLoader, self).__init__(fullname, fileobj, filename, etc)
 
@@ -266,10 +263,11 @@ else:
             """Same as `pkgutil.ImpLoader`, with an extra check for Hy
             source"""
             fullname = self._fix_name(fullname)
+            ext_type = self.etc[0]
             mod_type = self.etc[2]
             mod = None
             pkg_path = os.path.join(self.filename, '__init__.hy')
-            if mod_type == HY_SOURCE or (
+            if ext_type == '.hy' or (
                     mod_type == imp.PKG_DIRECTORY and
                     os.path.isfile(pkg_path)):
 
@@ -309,13 +307,13 @@ else:
         def _reopen(self):
             """Same as `pkgutil.ImpLoader`, with an extra check for Hy
             source"""
-            super(HyLoader, self)._reopen()
-
-            # Add the Hy case...
             if self.file and self.file.closed:
-                mod_type = self.etc[2]
-                if mod_type == HY_SOURCE:
+                ext_type = self.etc[0]
+                if ext_type == '.hy':
                     self.file = io.open(self.filename, 'rU', encoding='utf-8')
+                else:
+                    super(HyLoader, self)._reopen()
+
 
         def byte_compile_hy(self, fullname=None):
             fullname = self._fix_name(fullname)
@@ -344,8 +342,8 @@ else:
             """Same as `pkgutil.ImpLoader`, with an extra check for Hy
             source"""
             fullname = self._fix_name(fullname)
-            mod_type = self.etc[2]
-            if mod_type == HY_SOURCE:
+            ext_type = self.etc[0]
+            if ext_type == '.hy':
                 # Looks like we have to manually check for--and update--
                 # the bytecode.
                 t_py = long(os.stat(self.filename).st_mtime)
@@ -370,40 +368,10 @@ else:
 
             return self.code
 
-        def get_source(self, fullname=None):
-            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
-            source"""
-            fullname = self._fix_name(fullname)
-            mod_type = self.etc[2]
-            if self.source is None and mod_type == HY_SOURCE:
-                self._reopen()
-                try:
-                    self.source = self.file.read()
-                finally:
-                    self.file.close()
-
-            if self.source is None:
-                super(HyLoader, self).get_source(fullname=fullname)
-
-            return self.source
-
-        def get_filename(self, fullname=None):
-            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
-            source"""
-            fullname = self._fix_name(fullname)
-            if self.etc[2] == HY_SOURCE:
-                return self.filename
-
-            if self.filename is None:
-                filename = super(HyLoader, self).get_filename(fullname=fullname)
-
-            return filename
-
         def _get_delegate(self):
             return HyImporter(self.filename).find_module('__init__')
 
     class HyImporter(ImpImporter, object):
-
         def __init__(self, path=None):
             # We need to be strict about the types of files this importer will
             # handle.  To start, if the path is not the current directory in
@@ -444,7 +412,7 @@ else:
                     entry.endswith('.hy')):
                     file_path = entry
                     fileobj = io.open(file_path, 'rU', encoding='utf-8')
-                    etc = ('.hy', 'U', HY_SOURCE)
+                    etc = ('.hy', 'U', imp.PY_SOURCE)
                     break
                 else:
                     file_path = os.path.join(entry, subname)
@@ -457,7 +425,7 @@ else:
                     file_path = file_path + '.hy'
                     if os.path.isfile(file_path):
                         fileobj = io.open(file_path, 'rU', encoding='utf-8')
-                        etc = ('.hy', 'U', HY_SOURCE)
+                        etc = ('.hy', 'U', imp.PY_SOURCE)
                         break
             else:
                 try:
@@ -465,7 +433,6 @@ else:
                 except (ImportError, IOError):
                     return None
 
-
             return HyLoader(fullname, file_path, fileobj, etc)
 
     sys.path_hooks.append(HyImporter)

From 5d325a515640da2db3b830ade9fff64355042165 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Mon, 27 Aug 2018 00:28:02 -0500
Subject: [PATCH 070/223] Add a test for module docstrings

---
 hy/importer.py                        | 11 +++++------
 tests/importer/test_importer.py       | 13 +++++++++++++
 tests/resources/importer/docstring.hy |  5 +++++
 3 files changed, 23 insertions(+), 6 deletions(-)
 create mode 100644 tests/resources/importer/docstring.hy

diff --git a/hy/importer.py b/hy/importer.py
index 5cf7435..1ed6e7e 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -256,8 +256,8 @@ else:
 
         def exec_module(self, module, fullname=None):
             fullname = self._fix_name(fullname)
-            ast = self.get_code(fullname)
-            eval(ast, module.__dict__)
+            code = self.get_code(fullname)
+            eval(code, module.__dict__)
 
         def load_module(self, fullname=None):
             """Same as `pkgutil.ImpLoader`, with an extra check for Hy
@@ -287,7 +287,6 @@ else:
                     mod.__file__ = self.get_filename(fullname)
                     mod.__package__ = '.'.join(fullname.split('.')[:-1])
 
-                # TODO: Set `mod.__doc__`.
                 mod.__name__ = fullname
 
                 self.exec_module(mod, fullname=fullname)
@@ -314,7 +313,6 @@ else:
                 else:
                     super(HyLoader, self)._reopen()
 
-
         def byte_compile_hy(self, fullname=None):
             fullname = self._fix_name(fullname)
             if fullname is None:
@@ -322,8 +320,9 @@ else:
             try:
                 hy_source = self.get_source(fullname)
                 hy_tree = hy_parse(hy_source)
-                ast = hy_compile(hy_tree, fullname)
-                code = compile(ast, self.filename, 'exec',
+                hy_ast = hy_compile(hy_tree, fullname)
+
+                code = compile(hy_ast, self.filename, 'exec',
                                hy_ast_compile_flags)
             except (HyTypeError, LexException) as e:
                 if e.source is None:
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index bb92bce..900da03 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -253,3 +253,16 @@ def test_shadowed_basename():
         assert some_mod.ext == 'hy'
     finally:
         sys.path.pop(0)
+
+
+def test_docstring():
+    """Make sure a module's docstring is loaded."""
+    sys.path.insert(0, os.path.realpath('tests/resources/importer'))
+    try:
+        mod = importlib.import_module('docstring')
+        expected_doc = ("This module has a docstring.\n\n"
+                        "It covers multiple lines, too!\n")
+        assert mod.__doc__ == expected_doc
+        assert mod.a == 1
+    finally:
+        sys.path.pop(0)
diff --git a/tests/resources/importer/docstring.hy b/tests/resources/importer/docstring.hy
new file mode 100644
index 0000000..87d18b2
--- /dev/null
+++ b/tests/resources/importer/docstring.hy
@@ -0,0 +1,5 @@
+"This module has a docstring.
+
+It covers multiple lines, too!
+"
+(setv a 1)

From a9fca8001e3eeb2a53eb52cfa63c1116ee6b4a1b Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Mon, 27 Aug 2018 18:30:40 -0500
Subject: [PATCH 071/223] Fix AST handling of docstrings and __future__
 ordering

This closes hylang/hy#1367 and closes hylang/hy#1540
---
 hy/compiler.py              | 14 +++++++++++++-
 tests/compilers/test_ast.py | 22 ++++++++++++++++++++++
 tests/resources/pydemo.hy   |  1 +
 tests/test_hy2py.py         |  3 +++
 4 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 575c5a9..bd43cf7 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1719,7 +1719,19 @@ def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
     if not get_expr:
         result += result.expr_as_stmt()
 
-    body = compiler.imports_as_stmts(tree) + result.stmts
+    body = []
+
+    # Pull out a single docstring and prepend to the resulting body.
+    if (len(result.stmts) > 0 and
+        issubclass(root, ast.Module) and
+        isinstance(result.stmts[0], ast.Expr) and
+        isinstance(result.stmts[0].value, ast.Str)):
+
+        body += [result.stmts.pop(0)]
+
+    body += sorted(compiler.imports_as_stmts(tree) + result.stmts,
+                   key=lambda a: not (isinstance(a, ast.ImportFrom) and
+                                      a.module == '__future__'))
 
     ret = root(body=body)
 
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 3a1e52a..cb3ab7b 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -664,3 +664,25 @@ def test_ast_bad_yield_from():
 def test_eval_generator_with_return():
     """Ensure generators with a return statement works."""
     can_eval("(fn [] (yield 1) (yield 2) (return))")
+
+
+def test_futures_imports():
+    """Make sure __future__ imports go first, especially when builtins are
+    automatically added (e.g. via use of a builtin name like `name`)."""
+    hy_ast = can_compile((
+        '(import [__future__ [print_function]])\n'
+        '(import sys)\n'
+        '(setv name [1 2])'
+        '(print (first name))'))
+
+    assert hy_ast.body[0].module == '__future__'
+    assert hy_ast.body[1].module == 'hy.core.language'
+
+    hy_ast = can_compile((
+        '(import sys)\n'
+        '(import [__future__ [print_function]])\n'
+        '(setv name [1 2])'
+        '(print (first name))'))
+
+    assert hy_ast.body[0].module == '__future__'
+    assert hy_ast.body[1].module == 'hy.core.language'
diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
index b5e9cd7..d8a2447 100644
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -4,6 +4,7 @@
 
 ;; This Hy module is intended to concisely demonstrate all of
 ;; Python's major syntactic features for the purpose of testing hy2py.
+"This is a module docstring."
 
 (setv mystring (* "foo" 3))
 
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
index a0f1188..6bb0786 100644
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -24,6 +24,9 @@ def test_hy2py_import(tmpdir):
 
 def assert_stuff(m):
 
+    # This makes sure that automatically imported builtins go after docstrings.
+    assert m.__doc__ == u'This is a module docstring.'
+
     assert m.mystring == "foofoofoo"
 
     assert m.long_string == u"This is a very long string literal, which would surely exceed any limitations on how long a line or a string literal can be. The string literal alone exceeds 256 characters. It also has a character outside the Basic Multilingual Plane: 😂. Here's a double quote: \". Here are some escaped newlines:\n\n\nHere is a literal newline:\nCall me Ishmael. Some years ago—never mind how long precisely—having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world. It is a way I have of driving off the spleen and regulating the circulation. Whenever I find myself growing grim about the mouth; whenever it is a damp, drizzly November in my soul; whenever I find myself involuntarily pausing before coffin warehouses, and bringing up the rear of every funeral I meet; and especially whenever my hypos get such an upper hand of me, that it requires a strong moral principle to prevent me from deliberately stepping into the street, and methodically knocking people’s hats off—then, I account it high time to get to sea as soon as I can. This is my substitute for pistol and ball. With a philosophical flourish Cato throws himself upon his sword; I quietly take to the ship. There is nothing surprising in this. If they but knew it, almost all men in their degree, some time or other, cherish very nearly the same feelings towards the ocean with me."

From f4e5f02b3e7b297d6e4ffa2567fd025a021d3127 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Tue, 28 Aug 2018 16:58:59 -0500
Subject: [PATCH 072/223] Updated NEWS

---
 NEWS.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 1fe1e41..5a67e80 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -22,6 +22,8 @@ Bug Fixes
   attribute access.
 * Fixed crashes on Windows when calling `hy-repr` on date and time
   objects.
+* Fixed errors from `from __future__ import ...` statements and missing Hy
+  module docstrings caused by automatic importing of Hy builtins.
 
 0.15.0
 ==============================

From c0c5c9c6993cdd3533fb3be8b7b5c80dd0d16af5 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Tue, 4 Sep 2018 22:19:40 -0500
Subject: [PATCH 073/223] Make cmdline Hy process unknown filetypes as Hy
 source

This change a Hy-preferring `runhy` that is used by cmdline Hy.  Standard
`runpy` is still patched so that it can run `.hy` files, but the default
behaviour for unknown filetypes is preserved (i.e. assume they are Python
source).

Closes hylang/hy#1677.
---
 hy/cmdline.py                |  4 +-
 hy/importer.py               | 89 +++++++++++++++++++++++++-----------
 tests/resources/no_extension |  2 +
 tests/test_bin.py            |  6 +++
 4 files changed, 73 insertions(+), 28 deletions(-)
 create mode 100644 tests/resources/no_extension

diff --git a/hy/cmdline.py b/hy/cmdline.py
index 08f6c0b..f38355b 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -19,7 +19,7 @@ import astor.code_gen
 import hy
 from hy.lex import LexException, PrematureEndOfInput, mangle
 from hy.compiler import HyTypeError, hy_compile
-from hy.importer import hy_eval, hy_parse
+from hy.importer import hy_eval, hy_parse, runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
@@ -352,7 +352,7 @@ def cmdline_handler(scriptname, argv):
 
             try:
                 sys.argv = options.args
-                runpy.run_path(filename, run_name='__main__')
+                runhy.run_path(filename, run_name='__main__')
                 return 0
             except FileNotFoundError as e:
                 print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
diff --git a/hy/importer.py b/hy/importer.py
index 1ed6e7e..1bc8573 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -11,12 +11,13 @@ import inspect
 import pkgutil
 import re
 import io
-import runpy
 import types
 import tempfile
 import importlib
 import __future__
 
+from functools import partial
+
 from hy.errors import HyTypeError
 from hy.compiler import hy_compile
 from hy.lex import tokenize, LexException
@@ -166,46 +167,62 @@ def cache_from_source(source_path):
         return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
 
 
-def _get_code_from_file(run_name, fname=None):
-    """A patch of `runpy._get_code_from_file` that will also compile Hy
-    code.
+def _hy_code_from_file(filename, loader_type=None):
+    """Use PEP-302 loader to produce code for a given Hy source file."""
+    full_fname = os.path.abspath(filename)
+    fname_path, fname_file = os.path.split(full_fname)
+    modname = os.path.splitext(fname_file)[0]
+    sys.path.insert(0, fname_path)
+    try:
+        if loader_type is None:
+            loader = pkgutil.get_loader(modname)
+        else:
+            loader = loader_type(modname, full_fname)
+        code = loader.get_code(modname)
+    finally:
+        sys.path.pop(0)
 
-    This version will read and cache bytecode for Hy files.  It operates
-    normally otherwise.
+    return code
+
+
+def _get_code_from_file(run_name, fname=None,
+                        hy_src_check=lambda x: x.endswith('.hy')):
+    """A patch of `runpy._get_code_from_file` that will also run and cache Hy
+    code.
     """
     if fname is None and run_name is not None:
         fname = run_name
 
-    if fname.endswith('.hy'):
-        full_fname = os.path.abspath(fname)
-        fname_path, fname_file = os.path.split(full_fname)
-        modname = os.path.splitext(fname_file)[0]
-        sys.path.insert(0, fname_path)
-        try:
-            loader = pkgutil.get_loader(modname)
-            code = loader.get_code(modname)
-        finally:
-            sys.path.pop(0)
-    else:
-        with open(fname, "rb") as f:
-            code = pkgutil.read_code(f)
-        if code is None:
+    # Check for bytecode first.  (This is what the `runpy` version does!)
+    with open(fname, "rb") as f:
+        code = pkgutil.read_code(f)
+
+    if code is None:
+        if hy_src_check(fname):
+            code = _hy_code_from_file(fname, loader_type=HyLoader)
+        else:
+            # Try normal source
             with open(fname, "rb") as f:
+                # This code differs from `runpy`'s only in that we
+                # force decoding into UTF-8.
                 source = f.read().decode('utf-8')
             code = compile(source, fname, 'exec')
 
     return (code, fname) if PY3 else code
 
 
-_runpy_get_code_from_file = runpy._get_code_from_file
-runpy._get_code_from_file = _get_code_from_file
-
 if PY3:
     importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy')
     _py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code
 
+    def _could_be_hy_src(filename):
+        return (os.path.isfile(filename) and
+            (filename.endswith('.hy') or
+             not any(filename.endswith(ext)
+                     for ext in importlib.machinery.SOURCE_SUFFIXES[1:])))
+
     def _hy_source_to_code(self, data, path, _optimize=-1):
-        if os.path.isfile(path) and path.endswith('.hy'):
+        if _could_be_hy_src(path):
             source = data.decode("utf-8")
             try:
                 hy_tree = hy_parse(source)
@@ -242,12 +259,17 @@ else:
 
     from pkgutil import ImpImporter, ImpLoader
 
+    def _could_be_hy_src(filename):
+        return (filename.endswith('.hy') or
+                (os.path.isfile(filename) and
+                 not any(filename.endswith(s[0]) for s in imp.get_suffixes())))
+
     class HyLoader(ImpLoader, object):
         def __init__(self, fullname, filename, fileobj=None, etc=None):
             """This constructor is designed for some compatibility with
             SourceFileLoader."""
             if etc is None and filename is not None:
-                if filename.endswith('.hy'):
+                if _could_be_hy_src(filename):
                     etc = ('.hy', 'U', imp.PY_SOURCE)
                     if fileobj is None:
                         fileobj = io.open(filename, 'rU', encoding='utf-8')
@@ -477,7 +499,7 @@ else:
 
             try:
                 flags = None
-                if filename.endswith('.hy'):
+                if _could_be_hy_src(filename):
                     hy_tree = hy_parse(source_str)
                     source = hy_compile(hy_tree, '<hyc_compile>')
                     flags = hy_ast_compile_flags
@@ -530,3 +552,18 @@ else:
         return cfile
 
     py_compile.compile = hyc_compile
+
+
+# We create a separate version of runpy, "runhy", that prefers Hy source over
+# Python.
+runhy = importlib.import_module('runpy')
+
+runhy._get_code_from_file = partial(_get_code_from_file,
+                                    hy_src_check=_could_be_hy_src)
+
+del sys.modules['runpy']
+
+runpy = importlib.import_module('runpy')
+
+_runpy_get_code_from_file = runpy._get_code_from_file
+runpy._get_code_from_file = _get_code_from_file
diff --git a/tests/resources/no_extension b/tests/resources/no_extension
new file mode 100644
index 0000000..3d52066
--- /dev/null
+++ b/tests/resources/no_extension
@@ -0,0 +1,2 @@
+#!/usr/bin/env hy
+(print "This Should Still Work")
\ No newline at end of file
diff --git a/tests/test_bin.py b/tests/test_bin.py
index e642f53..58d1448 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -382,3 +382,9 @@ def test_bin_hy_module_no_main():
 def test_bin_hy_sys_executable():
     output, _ = run_cmd("hy -c '(do (import sys) (print sys.executable))'")
     assert output.strip().endswith('/hy')
+
+
+def test_bin_hy_file_no_extension():
+    """Confirm that a file with no extension is processed as Hy source"""
+    output, _ = run_cmd("hy tests/resources/no_extension")
+    assert "This Should Still Work" in output

From 96f99c29d11c49833a1b81ef09e013cc1a152c93 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 5 Sep 2018 17:48:58 -0500
Subject: [PATCH 074/223] Fix missing import in `doc` macro expansion

---
 hy/core/macros.hy          | 4 ++--
 tests/native_tests/core.hy | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 744e237..88b5de9 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -6,8 +6,6 @@
 ;;; These macros form the hy language
 ;;; They are automatically required in every module, except inside hy.core
 
-(import [importlib [import-module]])
-
 (import [hy.models [HyList HySymbol]])
 
 (defmacro as-> [head name &rest rest]
@@ -248,6 +246,7 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
    Use ``#doc foo`` instead for help with tag macro ``#foo``.
    Use ``(help foo)`` instead for help with runtime objects."
   `(try
+     (import [importlib [import-module]])
      (help (. (import-module "hy")
               macros
               _hy_macros
@@ -265,6 +264,7 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
 
    Gets help for a tag macro function available in this module."
   `(try
+     (import [importlib [import-module]])
      (help (. (import-module "hy")
               macros
               _hy_tag
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index 39eb698..496d9bb 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -682,3 +682,10 @@ result['y in globals'] = 'y' in globals()")
 (defn test-comment []
   (assert-none (comment <h1>This is merely a comment.</h1>
                         <p> Move along. (Nothing to see here.)</p>)))
+
+(defn test-doc [capsys]
+  (doc doc)
+  (setv out_err (.readouterr capsys))
+  (assert (.startswith (.strip (first out_err))
+            "Help on function (doc) in module hy.core.macros:"))
+  (assert (empty? (second out_err))))

From a9763b34cf358e95d3f88f4818c95568d181f5ad Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sat, 29 Sep 2018 20:46:14 -0500
Subject: [PATCH 075/223] Fix `sys.modules` for failed imports in Python 2.7

Newly imported modules with compile and/or run-time errors were not being
removed from `sys.modules`.  This commit modifies the Python 2.7 loader so that
it follows Python's failed-initial-import logic and removes the module from
`sys.modules`.
---
 hy/importer.py                  | 15 ++++++++++---
 tests/importer/test_importer.py | 39 +++++++++++++++++++++++++--------
 tests/resources/fails.hy        |  5 +++++
 3 files changed, 47 insertions(+), 12 deletions(-)
 create mode 100644 tests/resources/fails.hy

diff --git a/hy/importer.py b/hy/importer.py
index 1bc8573..18bab3f 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -293,7 +293,8 @@ else:
                     mod_type == imp.PKG_DIRECTORY and
                     os.path.isfile(pkg_path)):
 
-                if fullname in sys.modules:
+                was_in_sys = fullname in sys.modules
+                if was_in_sys:
                     mod = sys.modules[fullname]
                 else:
                     mod = sys.modules.setdefault(
@@ -311,7 +312,15 @@ else:
 
                 mod.__name__ = fullname
 
-                self.exec_module(mod, fullname=fullname)
+                try:
+                    self.exec_module(mod, fullname=fullname)
+                except Exception:
+                    # Follow Python 2.7 logic and only remove a new, bad
+                    # module; otherwise, leave the old--and presumably
+                    # good--module in there.
+                    if not was_in_sys:
+                        del sys.modules[fullname]
+                    raise
 
             if mod is None:
                 self._reopen()
@@ -385,7 +394,7 @@ else:
                     self.code = self.byte_compile_hy(fullname)
 
             if self.code is None:
-              super(HyLoader, self).get_code(fullname=fullname)
+                super(HyLoader, self).get_code(fullname=fullname)
 
             return self.code
 
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index 900da03..224670e 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -5,7 +5,6 @@
 import os
 import sys
 import ast
-import imp
 import tempfile
 import runpy
 import importlib
@@ -15,12 +14,16 @@ from fractions import Fraction
 import pytest
 
 import hy
-from hy._compat import bytes_type
 from hy.errors import HyTypeError
 from hy.lex import LexException
 from hy.compiler import hy_compile
 from hy.importer import hy_parse, HyLoader, cache_from_source
 
+try:
+    from importlib import reload
+except ImportError:
+    from imp import reload
+
 
 def test_basics():
     "Make sure the basics of the importer work"
@@ -85,6 +88,15 @@ def test_import_error_reporting():
     assert _import_error_test() is not None
 
 
+def test_import_error_cleanup():
+    "Failed initial imports should not leave dead modules in `sys.modules`."
+
+    with pytest.raises(hy.errors.HyMacroExpansionError):
+        importlib.import_module('tests.resources.fails')
+
+    assert 'tests.resources.fails' not in sys.modules
+
+
 @pytest.mark.skipif(sys.dont_write_bytecode,
                     reason="Bytecode generation is suppressed")
 def test_import_autocompiles():
@@ -127,7 +139,13 @@ def test_eval():
 
 
 def test_reload():
-    """Copied from CPython's `test_import.py`"""
+    """Generate a test module, confirm that it imports properly (and puts the
+    module in `sys.modules`), then modify the module so that it produces an
+    error when reloaded.  Next, fix the error, reload, and check that the
+    module is updated and working fine.  Rinse, repeat.
+
+    This test is adapted from CPython's `test_import.py`.
+    """
 
     def unlink(filename):
         os.unlink(source)
@@ -160,7 +178,7 @@ def test_reload():
             f.write("(setv b (// 20 0))")
 
         with pytest.raises(ZeroDivisionError):
-            imp.reload(mod)
+            reload(mod)
 
         # But we still expect the module to be in sys.modules.
         mod = sys.modules.get(TESTFN)
@@ -178,7 +196,7 @@ def test_reload():
             f.write("(setv a 11)")
             f.write("(setv b (// 20 1))")
 
-        imp.reload(mod)
+        reload(mod)
 
         mod = sys.modules.get(TESTFN)
         assert mod is not None
@@ -186,15 +204,17 @@ def test_reload():
         assert mod.a == 11
         assert mod.b == 20
 
-        # Now cause a LexException
+        # Now cause a `LexException`, and confirm that the good module and its
+        # contents stick around.
         unlink(source)
 
         with open(source, "w") as f:
+            # Missing paren...
             f.write("(setv a 11")
             f.write("(setv b (// 20 1))")
 
         with pytest.raises(LexException):
-            imp.reload(mod)
+            reload(mod)
 
         mod = sys.modules.get(TESTFN)
         assert mod is not None
@@ -209,7 +229,7 @@ def test_reload():
             f.write("(setv a 12)")
             f.write("(setv b (// 10 1))")
 
-        imp.reload(mod)
+        reload(mod)
 
         mod = sys.modules.get(TESTFN)
         assert mod is not None
@@ -219,8 +239,9 @@ def test_reload():
 
     finally:
         del sys.path[0]
+        if TESTFN in sys.modules:
+            del sys.modules[TESTFN]
         unlink(source)
-        del sys.modules[TESTFN]
 
 
 def test_circular():
diff --git a/tests/resources/fails.hy b/tests/resources/fails.hy
new file mode 100644
index 0000000..516fb8e
--- /dev/null
+++ b/tests/resources/fails.hy
@@ -0,0 +1,5 @@
+"This module produces an error when imported."
+(defmacro a-macro [x]
+  (+ x 1))
+
+(print (a-macro 'blah))

From 701db83ba942a541d50973a51e4363a1e2006e61 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 8 Nov 2018 22:46:01 -0600
Subject: [PATCH 076/223] Remove `get_arity`

This function wasn't being used anywhere.
---
 hy/inspect.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/hy/inspect.py b/hy/inspect.py
index ff972ec..f6672f4 100644
--- a/hy/inspect.py
+++ b/hy/inspect.py
@@ -11,9 +11,6 @@ try:
     # Otherwise fallback to the legacy getargspec.
     inspect.signature  # noqa
 except AttributeError:
-    def get_arity(fn):
-        return len(inspect.getargspec(fn)[0])
-
     def has_kwargs(fn):
         argspec = inspect.getargspec(fn)
         return argspec.keywords is not None
@@ -23,11 +20,6 @@ except AttributeError:
         return inspect.formatargspec(*argspec)
 
 else:
-    def get_arity(fn):
-        parameters = inspect.signature(fn).parameters
-        return sum(1 for param in parameters.values()
-                   if param.kind == param.POSITIONAL_OR_KEYWORD)
-
     def has_kwargs(fn):
         parameters = inspect.signature(fn).parameters
         return any(param.kind == param.VAR_KEYWORD

From 58003389c55c05eb15dbe630b3c2b9597be6ebeb Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sat, 20 Oct 2018 15:25:58 -0400
Subject: [PATCH 077/223] Integrate hy.inspect into hy.macros

It's compatibility code, and there's not a lot of it, and having a module with the same name as a standard module can be a bit troublesome.
---
 hy/inspect.py | 29 -----------------------------
 hy/macros.py  | 30 ++++++++++++++++++++++++++----
 2 files changed, 26 insertions(+), 33 deletions(-)
 delete mode 100644 hy/inspect.py

diff --git a/hy/inspect.py b/hy/inspect.py
deleted file mode 100644
index f6672f4..0000000
--- a/hy/inspect.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright 2018 the authors.
-# This file is part of Hy, which is free software licensed under the Expat
-# license. See the LICENSE.
-
-from __future__ import absolute_import
-
-import inspect
-
-try:
-    # Check if we have the newer inspect.signature available.
-    # Otherwise fallback to the legacy getargspec.
-    inspect.signature  # noqa
-except AttributeError:
-    def has_kwargs(fn):
-        argspec = inspect.getargspec(fn)
-        return argspec.keywords is not None
-
-    def format_args(fn):
-        argspec = inspect.getargspec(fn)
-        return inspect.formatargspec(*argspec)
-
-else:
-    def has_kwargs(fn):
-        parameters = inspect.signature(fn).parameters
-        return any(param.kind == param.VAR_KEYWORD
-                   for param in parameters.values())
-
-    def format_args(fn):
-        return str(inspect.signature(fn))
diff --git a/hy/macros.py b/hy/macros.py
index a86f043..787fef3 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -1,19 +1,41 @@
 # Copyright 2018 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
-import pkgutil
+import inspect
 import importlib
 
 from collections import defaultdict
 
 from hy._compat import PY3
-import hy.inspect
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
 from hy._compat import str_type
 
 from hy.errors import HyTypeError, HyMacroExpansionError
 
+try:
+    # Check if we have the newer inspect.signature available.
+    # Otherwise fallback to the legacy getargspec.
+    inspect.signature  # noqa
+except AttributeError:
+    def has_kwargs(fn):
+        argspec = inspect.getargspec(fn)
+        return argspec.keywords is not None
+
+    def format_args(fn):
+        argspec = inspect.getargspec(fn)
+        return inspect.formatargspec(*argspec)
+
+else:
+    def has_kwargs(fn):
+        parameters = inspect.signature(fn).parameters
+        return any(param.kind == param.VAR_KEYWORD
+                   for param in parameters.values())
+
+    def format_args(fn):
+        return str(inspect.signature(fn))
+
+
 CORE_MACROS = [
     "hy.core.bootstrap",
 ]
@@ -42,7 +64,7 @@ def macro(name):
     def _(fn):
         fn.__name__ = '({})'.format(name)
         try:
-            fn._hy_macro_pass_compiler = hy.inspect.has_kwargs(fn)
+            fn._hy_macro_pass_compiler = has_kwargs(fn)
         except Exception:
             # An exception might be raised if fn has arguments with
             # names that are invalid in Python.
@@ -139,7 +161,7 @@ def make_empty_fn_copy(fn):
         # can continue running. Unfortunately, the error message that might get
         # raised later on while expanding a macro might not make sense at all.
 
-        formatted_args = hy.inspect.format_args(fn)
+        formatted_args = format_args(fn)
         fn_str = 'lambda {}: None'.format(
             formatted_args.lstrip('(').rstrip(')'))
         empty_fn = eval(fn_str)

From 144a7fa240a07c8f13ca0c9ed5b7d106ad3de0d9 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Tue, 23 Oct 2018 14:41:20 -0400
Subject: [PATCH 078/223] Produce Python AST for `require` statements and skip
 self `require`s

Closes hylang/hy#1211.
---
 hy/cmdline.py                                 |  41 +--
 hy/compiler.py                                | 122 ++++++---
 hy/completer.py                               |  14 +-
 hy/contrib/walk.hy                            |   9 +-
 hy/core/language.hy                           |  12 +-
 hy/core/macros.hy                             |  28 +-
 hy/extra/reserved.hy                          |   2 +-
 hy/importer.py                                | 163 ++++++++---
 hy/macros.py                                  | 253 ++++++++++++------
 tests/importer/test_importer.py               |   4 +-
 tests/macros/test_macro_processor.py          |   8 +-
 tests/macros/test_tag_macros.py               |   1 +
 tests/resources/bin/circular_macro_require.hy |   8 +
 tests/resources/bin/require_and_eval.hy       |   3 +
 tests/test_bin.py                             |  35 +++
 15 files changed, 491 insertions(+), 212 deletions(-)
 create mode 100644 tests/resources/bin/circular_macro_require.hy
 create mode 100644 tests/resources/bin/require_and_eval.hy

diff --git a/hy/cmdline.py b/hy/cmdline.py
index f38355b..eeb5ccd 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -13,6 +13,7 @@ import io
 import importlib
 import py_compile
 import runpy
+import types
 
 import astor.code_gen
 
@@ -47,10 +48,26 @@ builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
 
-class HyREPL(code.InteractiveConsole):
+class HyREPL(code.InteractiveConsole, object):
     def __init__(self, spy=False, output_fn=None, locals=None,
                  filename="<input>"):
 
+        super(HyREPL, self).__init__(locals=locals,
+                                     filename=filename)
+
+        # Create a proper module for this REPL so that we can obtain it easily
+        # (e.g. using `importlib.import_module`).
+        # Also, make sure it's properly introduced to `sys.modules` and
+        # consistently use its namespace as `locals` from here on.
+        module_name = self.locals.get('__name__', '__console__')
+        self.module = sys.modules.setdefault(module_name,
+                                             types.ModuleType(module_name))
+        self.module.__dict__.update(self.locals)
+        self.locals = self.module.__dict__
+
+        # Load cmdline-specific macros.
+        require('hy.cmdline', module_name, assignments='ALL')
+
         self.spy = spy
 
         if output_fn is None:
@@ -65,9 +82,6 @@ class HyREPL(code.InteractiveConsole):
             else:
                 self.output_fn = __builtins__[mangle(output_fn)]
 
-        code.InteractiveConsole.__init__(self, locals=locals,
-                                         filename=filename)
-
         # Pre-mangle symbols for repl recent results: *1, *2, *3
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
@@ -102,8 +116,7 @@ class HyREPL(code.InteractiveConsole):
                     new_ast = ast.Module(main_ast.body +
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
-            value = hy_eval(do, self.locals, "__console__",
-                            ast_callback)
+            value = hy_eval(do, self.locals, self.module, ast_callback)
         except HyTypeError as e:
             if e.source is None:
                 e.source = source
@@ -181,8 +194,6 @@ def ideas_macro(ETname):
 
 """)])
 
-require("hy.cmdline", "__console__", assignments="ALL")
-require("hy.cmdline", "__main__", assignments="ALL")
 
 SIMPLE_TRACEBACKS = True
 
@@ -199,7 +210,8 @@ def pretty_error(func, *args, **kw):
 
 def run_command(source):
     tree = hy_parse(source)
-    pretty_error(hy_eval, tree, module_name="__main__")
+    require("hy.cmdline", "__main__", assignments="ALL")
+    pretty_error(hy_eval, tree, None, importlib.import_module('__main__'))
     return 0
 
 
@@ -208,13 +220,13 @@ def run_repl(hr=None, **kwargs):
     sys.ps1 = "=> "
     sys.ps2 = "... "
 
-    namespace = {'__name__': '__console__', '__doc__': ''}
+    if not hr:
+        hr = HyREPL(**kwargs)
+
+    namespace = hr.locals
 
     with completion(Completer(namespace)):
 
-        if not hr:
-            hr = HyREPL(locals=namespace, **kwargs)
-
         hr.interact("{appname} {version} using "
                     "{py}({build}) {pyversion} on {os}".format(
                         appname=hy.__appname__,
@@ -409,7 +421,6 @@ def hyc_main():
 # entry point for cmd line script "hy2py"
 def hy2py_main():
     import platform
-    module_name = "<STDIN>"
 
     options = dict(prog="hy2py", usage="%(prog)s [options] [FILE]",
                    formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -448,7 +459,7 @@ def hy2py_main():
         print()
         print()
 
-    _ast = pretty_error(hy_compile, hst, module_name)
+    _ast = pretty_error(hy_compile, hst, '__main__')
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))
diff --git a/hy/compiler.py b/hy/compiler.py
index bd43cf7..ceef84b 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -13,14 +13,16 @@ from hy.errors import HyCompileError, HyTypeError
 
 from hy.lex import mangle, unmangle
 
-import hy.macros
-from hy._compat import (
-    str_type, bytes_type, long_type, PY3, PY35, raise_empty)
-from hy.macros import require, macroexpand, tag_macroexpand
+from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
+                        PY35, raise_empty)
+from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 import hy.importer
 
 import traceback
 import importlib
+import inspect
+import pkgutil
+import types
 import ast
 import sys
 import copy
@@ -283,28 +285,45 @@ _stdlib = {}
 
 
 class HyASTCompiler(object):
+    """A Hy-to-Python AST compiler"""
 
-    def __init__(self, module_name):
+    def __init__(self, module):
+        """
+        Parameters
+        ----------
+        module: str or types.ModuleType
+            Module in which the Hy tree is evaluated.
+        """
         self.anon_var_count = 0
         self.imports = defaultdict(set)
-        self.module_name = module_name
         self.temp_if = None
+
+        if not inspect.ismodule(module):
+            module = importlib.import_module(module)
+
+        self.module = module
+        self.module_name = module.__name__
+
         self.can_use_stdlib = (
-            not module_name.startswith("hy.core")
-            or module_name == "hy.core.macros")
+            not self.module_name.startswith("hy.core")
+            or self.module_name == "hy.core.macros")
+
+        # Load stdlib macros into the module namespace.
+        load_macros(self.module)
+
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
         if self.can_use_stdlib and not _stdlib:
             # Populate _stdlib.
             import hy.core
-            for module in hy.core.STDLIB:
-                mod = importlib.import_module(module)
-                for e in map(ast_str, mod.EXPORTS):
+            for stdlib_module in hy.core.STDLIB:
+                mod = importlib.import_module(stdlib_module)
+                for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
                     if getattr(mod, e) is not getattr(builtins, e, ''):
                         # Don't bother putting a name in _stdlib if it
                         # points to a builtin with the same name. This
                         # prevents pointless imports.
-                        _stdlib[e] = module
+                        _stdlib[e] = stdlib_module
 
     def get_anon_var(self):
         self.anon_var_count += 1
@@ -1098,11 +1117,6 @@ class HyASTCompiler(object):
         brackets(SYM, sym(":as"), _symn) |
         brackets(SYM, brackets(many(_symn + maybe(sym(":as") + _symn)))))])
     def compile_import_or_require(self, expr, root, entries):
-        """
-        TODO for `require`: keep track of what we've imported in this run and
-        then "unimport" it after we've completed `thing' so that we don't
-        pollute other envs.
-        """
         ret = Result()
 
         for entry in entries:
@@ -1128,8 +1142,9 @@ class HyASTCompiler(object):
                 else:
                     assignments = [(k, v or k) for k, v in kids]
 
+            ast_module = ast_str(module, piecewise=True)
+
             if root == "import":
-                ast_module = ast_str(module, piecewise=True)
                 module = ast_module.lstrip(".")
                 level = len(ast_module) - len(module)
                 if assignments == "ALL" and prefix == "":
@@ -1150,10 +1165,23 @@ class HyASTCompiler(object):
                         for k, v in assignments]
                 ret += node(
                     expr, module=module or None, names=names, level=level)
-            else: # root == "require"
-                importlib.import_module(module)
-                require(module, self.module_name,
-                        assignments=assignments, prefix=prefix)
+
+            elif require(ast_module, self.module, assignments=assignments,
+                         prefix=prefix):
+                # Actually calling `require` is necessary for macro expansions
+                # occurring during compilation.
+                self.imports['hy.macros'].update([None])
+                # The `require` we're creating in AST is the same as above, but used at
+                # run-time (e.g. when modules are loaded via bytecode).
+                ret += self.compile(HyExpression([
+                    HySymbol('hy.macros.require'),
+                    HyString(ast_module),
+                    HySymbol('None'),
+                    HyKeyword('assignments'),
+                    (HyString("ALL") if assignments == "ALL" else
+                        [[HyString(k), HyString(v)] for k, v in assignments]),
+                    HyKeyword('prefix'),
+                    HyString(prefix)]).replace(expr))
 
         return ret
 
@@ -1484,7 +1512,8 @@ class HyASTCompiler(object):
                 [x for pair in attrs[0] for x in pair]).replace(attrs)))
 
         for e in body:
-            e = self.compile(self._rewire_init(macroexpand(e, self)))
+            e = self.compile(self._rewire_init(
+                macroexpand(e, self.module, self)))
             bodyr += e + e.expr_as_stmt()
 
         return bases + asty.ClassDef(
@@ -1520,20 +1549,16 @@ class HyASTCompiler(object):
         return self.compile(tag_macroexpand(
             HyString(mangle(tag)).replace(tag),
             arg,
-            self))
-
-    _namespaces = {}
+            self.module))
 
     @special(["eval-and-compile", "eval-when-compile"], [many(FORM)])
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
-        if self.module_name not in self._namespaces:
-            # Initialize a compile-time namespace for this module.
-            self._namespaces[self.module_name] = {
-                'hy': hy, '__name__': self.module_name}
+
         hy.importer.hy_eval(new_expr + body,
-                            self._namespaces[self.module_name],
-                            self.module_name)
+                            self.module.__dict__,
+                            self.module)
+
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
                 else Result())
@@ -1541,7 +1566,7 @@ class HyASTCompiler(object):
     @builds_model(HyExpression)
     def compile_expression(self, expr):
         # Perform macro expansions
-        expr = macroexpand(expr, self)
+        expr = macroexpand(expr, self.module, self)
         if not isinstance(expr, HyExpression):
             # Go through compile again if the type changed.
             return self.compile(expr)
@@ -1699,20 +1724,41 @@ class HyASTCompiler(object):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
-def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
+def hy_compile(tree, module, root=ast.Module, get_expr=False):
     """
-    Compile a HyObject tree into a Python AST Module.
+    Compile a Hy tree into a Python AST tree.
 
-    If `get_expr` is True, return a tuple (module, last_expression), where
-    `last_expression` is the.
+    Parameters
+    ----------
+    module: str or types.ModuleType
+        Module, or name of the module, in which the Hy tree is evaluated.
+
+    root: ast object, optional (ast.Module)
+        Root object for the Python AST tree.
+
+    get_expr: bool, optional (False)
+        If true, return a tuple with `(root_obj, last_expression)`.
+
+    Returns
+    -------
+    out : A Python AST tree
     """
 
+    if isinstance(module, string_types):
+        if module.startswith('<') and module.endswith('>'):
+            module = types.ModuleType(module)
+        else:
+            module = importlib.import_module(ast_str(module, piecewise=True))
+    if not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+
     tree = wrap_value(tree)
     if not isinstance(tree, HyObject):
         raise HyCompileError("`tree` must be a HyObject or capable of "
                              "being promoted to one")
 
-    compiler = HyASTCompiler(module_name)
+    compiler = HyASTCompiler(module)
     result = compiler.compile(tree)
     expr = result.force_expr
 
diff --git a/hy/completer.py b/hy/completer.py
index 7748c3d..9b7bb4f 100644
--- a/hy/completer.py
+++ b/hy/completer.py
@@ -39,13 +39,15 @@ class Completer(object):
         self.namespace = namespace
         self.path = [hy.compiler._special_form_compilers,
                      builtins.__dict__,
-                     hy.macros._hy_macros[None],
                      namespace]
-        self.tag_path = [hy.macros._hy_tag[None]]
-        if '__name__' in namespace:
-            module_name = namespace['__name__']
-            self.path.append(hy.macros._hy_macros[module_name])
-            self.tag_path.append(hy.macros._hy_tag[module_name])
+
+        self.tag_path = []
+
+        namespace.setdefault('__macros__', {})
+        namespace.setdefault('__tags__', {})
+
+        self.path.append(namespace['__macros__'])
+        self.tag_path.append(namespace['__tags__'])
 
     def attr_matches(self, text):
         # Borrowed from IPython's completer
diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index 4b6df7a..70fbca5 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -5,6 +5,7 @@
 
 (import [hy [HyExpression HyDict]]
         [functools [partial]]
+        [importlib [import-module]]
         [collections [OrderedDict]]
         [hy.macros [macroexpand :as mexpand]]
         [hy.compiler [HyASTCompiler]])
@@ -42,9 +43,11 @@
 
 (defn macroexpand-all [form &optional module-name]
   "Recursively performs all possible macroexpansions in form."
-  (setv module-name (or module-name (calling-module-name))
+  (setv module (or (and module-name
+                        (import-module module-name))
+                   (calling-module))
         quote-level [0]
-        ast-compiler (HyASTCompiler module-name))  ; TODO: make nonlocal after dropping Python2
+        ast-compiler (HyASTCompiler module))  ; TODO: make nonlocal after dropping Python2
   (defn traverse [form]
     (walk expand identity form))
   (defn expand [form]
@@ -68,7 +71,7 @@
               [(= (first form) (HySymbol "require"))
                (ast-compiler.compile form)
                (return)]
-              [True (traverse (mexpand form ast-compiler))])
+              [True (traverse (mexpand form module ast-compiler))])
         (if (coll? form)
             (traverse form)
             form)))
diff --git a/hy/core/language.hy b/hy/core/language.hy
index abb387f..5ada235 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -21,7 +21,7 @@
 (import [hy.models [HySymbol HyKeyword]])
 (import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
 (import [hy.compiler [HyASTCompiler]])
-(import [hy.importer [hy-eval :as eval]])
+(import [hy.importer [calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
@@ -295,12 +295,14 @@ Return series of accumulated sums (or other binary function results)."
 (defn macroexpand [form]
   "Return the full macro expansion of `form`."
   (import hy.macros)
-  (hy.macros.macroexpand form (HyASTCompiler (calling-module-name))))
+  (setv module (calling-module))
+  (hy.macros.macroexpand form module (HyASTCompiler module)))
 
 (defn macroexpand-1 [form]
   "Return the single step macro expansion of `form`."
   (import hy.macros)
-  (hy.macros.macroexpand-1 form (HyASTCompiler (calling-module-name))))
+  (setv module (calling-module))
+  (hy.macros.macroexpand-1 form module (HyASTCompiler module)))
 
 (defn merge-with [f &rest maps]
   "Return the map of `maps` joined onto the first via the function `f`.
@@ -467,8 +469,8 @@ Even objects with the __name__ magic will work."
     (or a b)))
 
 (setv EXPORTS
-  '[*map accumulate butlast calling-module-name chain coll? combinations
-    comp complement compress constantly count cycle dec distinct
+  '[*map accumulate butlast calling-module calling-module-name chain coll?
+    combinations comp complement compress constantly count cycle dec distinct
     disassemble drop drop-last drop-while empty? eval even? every? exec first
     filter flatten float? fraction gensym group-by identity inc input instance?
     integer integer? integer-char? interleave interpose islice iterable?
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 88b5de9..2f9154e 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -245,34 +245,10 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
 
    Use ``#doc foo`` instead for help with tag macro ``#foo``.
    Use ``(help foo)`` instead for help with runtime objects."
-  `(try
-     (import [importlib [import-module]])
-     (help (. (import-module "hy")
-              macros
-              _hy_macros
-              [__name__]
-              ['~symbol]))
-     (except [KeyError]
-       (help (. (import-module "hy")
-                macros
-                _hy_macros
-                [None]
-                ['~symbol])))))
+  `(help (.get __macros__ '~symbol None)))
 
 (deftag doc [symbol]
   "tag macro documentation
 
    Gets help for a tag macro function available in this module."
-  `(try
-     (import [importlib [import-module]])
-     (help (. (import-module "hy")
-              macros
-              _hy_tag
-              [__name__]
-              ['~symbol]))
-     (except [KeyError]
-       (help (. (import-module "hy")
-                macros
-                _hy_tag
-                [None]
-                ['~symbol])))))
+  `(help (.get __tags__ '~symbol None)))
diff --git a/hy/extra/reserved.hy b/hy/extra/reserved.hy
index 90ffee5..3de34ea 100644
--- a/hy/extra/reserved.hy
+++ b/hy/extra/reserved.hy
@@ -16,7 +16,7 @@
     (setv _cache (frozenset (map unmangle (+
       hy.core.language.EXPORTS
       hy.core.shadow.EXPORTS
-      (list (.keys (get hy.macros._hy_macros None)))
+      (list (.keys hy.core.macros.__macros__))
       keyword.kwlist
       (list (.keys hy.compiler._special_form_compilers))
       (list hy.compiler._bad_roots)))))))
diff --git a/hy/importer.py b/hy/importer.py
index 18bab3f..fdcb0be 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -17,9 +17,10 @@ import importlib
 import __future__
 
 from functools import partial
+from contextlib import contextmanager
 
 from hy.errors import HyTypeError
-from hy.compiler import hy_compile
+from hy.compiler import hy_compile, ast_str
 from hy.lex import tokenize, LexException
 from hy.models import HyExpression, HySymbol
 from hy._compat import string_types, PY3
@@ -29,6 +30,36 @@ hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
                         __future__.CO_FUTURE_PRINT_FUNCTION)
 
 
+def calling_module(n=1):
+    """Get the module calling, if available.
+
+    As a fallback, this will import a module using the calling frame's
+    globals value of `__name__`.
+
+    Parameters
+    ----------
+    n: int, optional
+        The number of levels up the stack from this function call.
+        The default is one level up.
+
+    Returns
+    -------
+    out: types.ModuleType
+        The module at stack level `n + 1` or `None`.
+    """
+    frame_up = inspect.stack(0)[n + 1][0]
+    module = inspect.getmodule(frame_up)
+    if module is None:
+        # This works for modules like `__main__`
+        module_name = frame_up.f_globals.get('__name__', None)
+        if module_name:
+            try:
+                module = importlib.import_module(module_name)
+            except ImportError:
+                pass
+    return module
+
+
 def ast_compile(ast, filename, mode):
     """Compile AST.
 
@@ -65,13 +96,9 @@ def hy_parse(source):
     return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
 
 
-def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
+def hy_eval(hytree, locals=None, module=None, ast_callback=None):
     """Evaluates a quoted expression and returns the value.
 
-    The optional second and third arguments specify the dictionary of globals
-    to use and the module name. The globals dictionary defaults to ``(local)``
-    and the module name defaults to the name of the current module.
-
     Examples
     --------
 
@@ -89,13 +116,15 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     hytree: a Hy expression tree
         Source code to parse.
 
-    namespace: dict, optional
-        Namespace in which to evaluate the Hy tree.  Defaults to the calling
-        frame.
+    locals: dict, optional
+        Local environment in which to evaluate the Hy tree.  Defaults to the
+        calling frame.
 
-    module_name: str, optional
-        Name of the module to which the Hy tree is assigned.  Defaults to
-        the calling frame's module, if any, and '__eval__' otherwise.
+    module: str or types.ModuleType, optional
+        Module, or name of the module, to which the Hy tree is assigned and
+        the global values are taken.
+        Defaults to the calling frame's module, if any, and '__eval__'
+        otherwise.
 
     ast_callback: callable, optional
         A callback that is passed the Hy compiled tree and resulting
@@ -105,19 +134,23 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     Returns
     -------
     out : Result of evaluating the Hy compiled tree.
-
     """
-    if namespace is None:
+    if module is None:
+        module = calling_module()
+
+    if isinstance(module, string_types):
+        module = importlib.import_module(ast_str(module, piecewise=True))
+    elif not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    if locals is None:
         frame = inspect.stack()[1][0]
-        namespace = inspect.getargvalues(frame).locals
-    if module_name is None:
-        m = inspect.getmodule(inspect.stack()[1][0])
-        module_name = '__eval__' if m is None else m.__name__
+        locals = inspect.getargvalues(frame).locals
 
-    if not isinstance(module_name, string_types):
-        raise TypeError("Module name must be a string")
+    if not isinstance(locals, dict):
+        raise TypeError("Locals must be a dictionary")
 
-    _ast, expr = hy_compile(hytree, module_name, get_expr=True)
+    _ast, expr = hy_compile(hytree, module, get_expr=True)
 
     # Spoof the positions in the generated ast...
     for node in ast.walk(_ast):
@@ -131,14 +164,13 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
     if ast_callback:
         ast_callback(_ast, expr)
 
-    if not isinstance(namespace, dict):
-        raise TypeError("Globals must be a dictionary")
+    globals = module.__dict__
 
     # Two-step eval: eval() the body of the exec call
-    eval(ast_compile(_ast, "<eval_body>", "exec"), namespace)
+    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
 
     # Then eval the expression context and return that
-    return eval(ast_compile(expr, "<eval>", "eval"), namespace)
+    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
 
 
 def cache_from_source(source_path):
@@ -167,6 +199,52 @@ def cache_from_source(source_path):
         return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
 
 
+@contextmanager
+def loader_module_obj(loader):
+    """Use the module object associated with a loader.
+
+    This is intended to be used by a loader object itself, and primarily as a
+    work-around for attempts to get module and/or file code from a loader
+    without actually creating a module object.  Since Hy currently needs the
+    module object for macro importing, expansion, and whatnot, using this will
+    reconcile Hy with such attempts.
+
+    For example, if we're first compiling a Hy script starting from
+    `runpy.run_path`, the Hy compiler will need a valid module object in which
+    to run, but, given the way `runpy.run_path` works, there might not be one
+    yet (e.g. `__main__` for a .hy file).  We compensate by properly loading
+    the module here.
+
+    The function `inspect.getmodule` has a hidden-ish feature that returns
+    modules using their associated filenames (via `inspect.modulesbyfile`),
+    and, since the Loaders (and their delegate Loaders) carry a filename/path
+    associated with the parent package, we use it as a more robust attempt to
+    obtain an existing module object.
+
+    When no module object is found, a temporary, minimally sufficient module
+    object is created for the duration of the `with` body.
+    """
+    tmp_mod = False
+
+    try:
+        module = inspect.getmodule(None, _filename=loader.path)
+    except KeyError:
+        module = None
+
+    if module is None:
+        tmp_mod = True
+        module = sys.modules.setdefault(loader.name,
+                                        types.ModuleType(loader.name))
+        module.__file__ = loader.path
+        module.__name__ = loader.name
+
+    try:
+        yield module
+    finally:
+        if tmp_mod:
+            del sys.modules[loader.name]
+
+
 def _hy_code_from_file(filename, loader_type=None):
     """Use PEP-302 loader to produce code for a given Hy source file."""
     full_fname = os.path.abspath(filename)
@@ -226,7 +304,8 @@ if PY3:
             source = data.decode("utf-8")
             try:
                 hy_tree = hy_parse(source)
-                data = hy_compile(hy_tree, self.name)
+                with loader_module_obj(self) as module:
+                    data = hy_compile(hy_tree, module)
             except (HyTypeError, LexException) as e:
                 if e.source is None:
                     e.source = source
@@ -276,6 +355,15 @@ else:
 
             super(HyLoader, self).__init__(fullname, fileobj, filename, etc)
 
+        def __getattr__(self, item):
+            # We add these for Python >= 3.4 Loader interface compatibility.
+            if item == 'path':
+                return self.filename
+            elif item == 'name':
+                return self.fullname
+            else:
+                return super(HyLoader, self).__getattr__(item)
+
         def exec_module(self, module, fullname=None):
             fullname = self._fix_name(fullname)
             code = self.get_code(fullname)
@@ -283,7 +371,7 @@ else:
 
         def load_module(self, fullname=None):
             """Same as `pkgutil.ImpLoader`, with an extra check for Hy
-            source"""
+            source and the option to not run `self.exec_module`."""
             fullname = self._fix_name(fullname)
             ext_type = self.etc[0]
             mod_type = self.etc[2]
@@ -298,7 +386,7 @@ else:
                     mod = sys.modules[fullname]
                 else:
                     mod = sys.modules.setdefault(
-                        fullname, imp.new_module(fullname))
+                        fullname, types.ModuleType(fullname))
 
                 # TODO: Should we set these only when not in `sys.modules`?
                 if mod_type == imp.PKG_DIRECTORY:
@@ -351,7 +439,8 @@ else:
             try:
                 hy_source = self.get_source(fullname)
                 hy_tree = hy_parse(hy_source)
-                hy_ast = hy_compile(hy_tree, fullname)
+                with loader_module_obj(self) as module:
+                    hy_ast = hy_compile(hy_tree, module)
 
                 code = compile(hy_ast, self.filename, 'exec',
                                hy_ast_compile_flags)
@@ -363,7 +452,7 @@ else:
 
             if not sys.dont_write_bytecode:
                 try:
-                    hyc_compile(code)
+                    hyc_compile(code, module=fullname)
                 except IOError:
                     pass
             return code
@@ -470,7 +559,8 @@ else:
 
     _py_compile_compile = py_compile.compile
 
-    def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False):
+    def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False,
+                    module=None):
         """Write a Hy file, or code object, to pyc.
 
         This is a patched version of Python 2.7's `py_compile.compile`.
@@ -489,6 +579,9 @@ else:
             The filename to use for compile-time errors.
         doraise : bool, default False
             If `True` raise compilation exceptions; otherwise, ignore them.
+        module : str or types.ModuleType, optional
+            The module, or module name, in which the Hy tree is expanded.
+            Default is the caller's module.
 
         Returns
         -------
@@ -510,7 +603,13 @@ else:
                 flags = None
                 if _could_be_hy_src(filename):
                     hy_tree = hy_parse(source_str)
-                    source = hy_compile(hy_tree, '<hyc_compile>')
+
+                    if module is None:
+                        module = inspect.getmodule(inspect.stack()[1][0])
+                    elif not inspect.ismodule(module):
+                        module = importlib.import_module(module)
+
+                    source = hy_compile(hy_tree, module)
                     flags = hy_ast_compile_flags
 
                 codeobject = compile(source, dfile or filename, 'exec', flags)
diff --git a/hy/macros.py b/hy/macros.py
index 787fef3..f3b64ff 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -1,15 +1,13 @@
 # Copyright 2018 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
-import inspect
 import importlib
+import inspect
+import pkgutil
 
-from collections import defaultdict
-
-from hy._compat import PY3
+from hy._compat import PY3, string_types
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
-from hy._compat import str_type
 
 from hy.errors import HyTypeError, HyMacroExpansionError
 
@@ -44,21 +42,9 @@ EXTRA_MACROS = [
     "hy.core.macros",
 ]
 
-_hy_macros = defaultdict(dict)
-_hy_tag = defaultdict(dict)
-
 
 def macro(name):
     """Decorator to define a macro called `name`.
-
-    This stores the macro `name` in the namespace for the module where it is
-    defined.
-
-    If the module where it is defined is in `hy.core`, then the macro is stored
-    in the default `None` namespace.
-
-    This function is called from the `defmacro` special form in the compiler.
-
     """
     name = mangle(name)
     def _(fn):
@@ -70,88 +56,190 @@ def macro(name):
             # names that are invalid in Python.
             fn._hy_macro_pass_compiler = False
 
-        module_name = fn.__module__
-        if module_name.startswith("hy.core"):
-            module_name = None
-        _hy_macros[module_name][name] = fn
+        module = inspect.getmodule(fn)
+        module_macros = module.__dict__.setdefault('__macros__', {})
+        module_macros[name] = fn
+
         return fn
     return _
 
 
 def tag(name):
     """Decorator to define a tag macro called `name`.
-
-    This stores the macro `name` in the namespace for the module where it is
-    defined.
-
-    If the module where it is defined is in `hy.core`, then the macro is stored
-    in the default `None` namespace.
-
-    This function is called from the `deftag` special form in the compiler.
-
     """
     def _(fn):
         _name = mangle('#{}'.format(name))
+
         if not PY3:
             _name = _name.encode('UTF-8')
+
         fn.__name__ = _name
-        module_name = fn.__module__
+
+        module = inspect.getmodule(fn)
+
+        module_name = module.__name__
         if module_name.startswith("hy.core"):
             module_name = None
-        _hy_tag[module_name][mangle(name)] = fn
+
+        module_tags = module.__dict__.setdefault('__tags__', {})
+        module_tags[mangle(name)] = fn
 
         return fn
     return _
 
 
+def _same_modules(source_module, target_module):
+    """Compare the filenames associated with the given modules names.
+
+    This tries to not actually load the modules.
+    """
+    if not (source_module or target_module):
+        return False
+
+    if target_module == source_module:
+        return True
+
+    def _get_filename(module):
+        filename = None
+        try:
+            if not inspect.ismodule(module):
+                loader = pkgutil.get_loader(module)
+                if loader:
+                    filename = loader.get_filename()
+            else:
+                filename = inspect.getfile(module)
+        except (TypeError, ImportError):
+            pass
+
+        return filename
+
+    source_filename = _get_filename(source_module)
+    target_filename = _get_filename(target_module)
+
+    return (source_filename and target_filename and
+            source_filename == target_filename)
+
+
 def require(source_module, target_module, assignments, prefix=""):
-    """Load macros from `source_module` in the namespace of
-    `target_module`. `assignments` maps old names to new names, or
-    should be the string "ALL". If `prefix` is nonempty, it is
-    prepended to the name of each imported macro. (This means you get
-    macros named things like "mymacromodule.mymacro", which looks like
-    an attribute of a module, although it's actually just a symbol
-    with a period in its name.)
+    """Load macros from one module into the namespace of another.
 
     This function is called from the `require` special form in the compiler.
 
+    Parameters
+    ----------
+    source_module: str or types.ModuleType
+        The module from which macros are to be imported.
+
+    target_module: str, types.ModuleType or None
+        The module into which the macros will be loaded.  If `None`, then
+        the caller's namespace.
+        The latter is useful during evaluation of generated AST/bytecode.
+
+    assignments: str or list of tuples of strs
+        The string "ALL" or a list of macro name and alias pairs.
+
+    prefix: str, optional ("")
+        If nonempty, its value is prepended to the name of each imported macro.
+        This allows one to emulate namespaced macros, like
+        "mymacromodule.mymacro", which looks like an attribute of a module.
+
+    Returns
+    -------
+    out: boolean
+        Whether or not macros and tags were actually transferred.
     """
 
-    seen_names = set()
+    if target_module is None:
+        parent_frame = inspect.stack()[1][0]
+        target_namespace = parent_frame.f_globals
+        target_module = target_namespace.get('__name__', None)
+    elif isinstance(target_module, string_types):
+        target_module = importlib.import_module(target_module)
+        target_namespace = target_module.__dict__
+    elif inspect.ismodule(target_module):
+        target_namespace = target_module.__dict__
+    else:
+        raise TypeError('`target_module` is not a recognized type: {}'.format(
+            type(target_module)))
+
+    # Let's do a quick check to make sure the source module isn't actually
+    # the module being compiled (e.g. when `runpy` executes a module's code
+    # in `__main__`).
+    # We use the module's underlying filename for this (when they exist), since
+    # it's the most "fixed" attribute.
+    if _same_modules(source_module, target_module):
+        return False
+
+    if not inspect.ismodule(source_module):
+        source_module = importlib.import_module(source_module)
+
+    source_macros = source_module.__dict__.setdefault('__macros__', {})
+    source_tags = source_module.__dict__.setdefault('__tags__', {})
+
+    if len(source_module.__macros__) + len(source_module.__tags__) == 0:
+        if assignments != "ALL":
+            raise ImportError('The module {} has no macros or tags'.format(
+                source_module))
+        else:
+            return False
+
+    target_macros = target_namespace.setdefault('__macros__', {})
+    target_tags = target_namespace.setdefault('__tags__', {})
+
     if prefix:
         prefix += "."
-    if assignments != "ALL":
-        assignments = {mangle(str_type(k)): v for k, v in assignments}
 
-    for d in _hy_macros, _hy_tag:
-        for name, macro in d[source_module].items():
-            seen_names.add(name)
-            if assignments == "ALL":
-                d[target_module][mangle(prefix + name)] = macro
-            elif name in assignments:
-                d[target_module][mangle(prefix + assignments[name])] = macro
+    if assignments == "ALL":
+        # Only add macros/tags created in/by the source module.
+        name_assigns = [(n, n) for n, f in source_macros.items()
+                        if inspect.getmodule(f) == source_module]
+        name_assigns += [(n, n) for n, f in source_tags.items()
+                         if inspect.getmodule(f) == source_module]
+    else:
+        # If one specifically requests a macro/tag not created in the source
+        # module, I guess we allow it?
+        name_assigns = assignments
 
-    if assignments != "ALL":
-        unseen = frozenset(assignments.keys()).difference(seen_names)
-        if unseen:
-            raise ImportError("cannot require names: " + repr(list(unseen)))
+    for name, alias in name_assigns:
+        _name = mangle(name)
+        alias = mangle(prefix + alias)
+        if _name in source_module.__macros__:
+            target_macros[alias] = source_macros[_name]
+        elif _name in source_module.__tags__:
+            target_tags[alias] = source_tags[_name]
+        else:
+            raise ImportError('Could not require name {} from {}'.format(
+                _name, source_module))
+
+    return True
 
 
-def load_macros(module_name):
+def load_macros(module):
     """Load the hy builtin macros for module `module_name`.
 
     Modules from `hy.core` can only use the macros from CORE_MACROS.
     Other modules get the macros from CORE_MACROS and EXTRA_MACROS.
-
     """
-    for module in CORE_MACROS:
-        importlib.import_module(module)
+    builtin_macros = CORE_MACROS
 
-    if module_name.startswith("hy.core"):
-        return
+    if not module.__name__.startswith("hy.core"):
+        builtin_macros += EXTRA_MACROS
 
-    for module in EXTRA_MACROS:
-        importlib.import_module(module)
+    module_macros = module.__dict__.setdefault('__macros__', {})
+    module_tags = module.__dict__.setdefault('__tags__', {})
+
+    for builtin_mod_name in builtin_macros:
+        builtin_mod = importlib.import_module(builtin_mod_name)
+
+        # Make sure we don't overwrite macros in the module.
+        if hasattr(builtin_mod, '__macros__'):
+            module_macros.update({k: v
+                                  for k, v in builtin_mod.__macros__.items()
+                                  if k not in module_macros})
+        if hasattr(builtin_mod, '__tags__'):
+            module_tags.update({k: v
+                                for k, v in builtin_mod.__tags__.items()
+                                if k not in module_tags})
 
 
 def make_empty_fn_copy(fn):
@@ -174,14 +262,17 @@ def make_empty_fn_copy(fn):
     return empty_fn
 
 
-def macroexpand(tree, compiler, once=False):
+def macroexpand(tree, module, compiler=None, once=False):
     """Expand the toplevel macros for the `tree`.
 
-    Load the macros from the given `compiler.module_name`, then expand the
+    Load the macros from the given `module`, then expand the
     (top-level) macros in `tree` until we no longer can.
-
     """
-    load_macros(compiler.module_name)
+    if not inspect.ismodule(module):
+        module = importlib.import_module(module)
+
+    assert not compiler or compiler.module == module
+
     while True:
 
         if not isinstance(tree, HyExpression) or tree == []:
@@ -192,24 +283,27 @@ def macroexpand(tree, compiler, once=False):
             break
 
         fn = mangle(fn)
-        m = _hy_macros[compiler.module_name].get(fn) or _hy_macros[None].get(fn)
+        m = module.__macros__.get(fn, None)
         if not m:
             break
 
         opts = {}
         if m._hy_macro_pass_compiler:
+            if compiler is None:
+                from hy.compiler import HyASTCompiler
+                compiler = HyASTCompiler(module)
             opts['compiler'] = compiler
 
         try:
             m_copy = make_empty_fn_copy(m)
-            m_copy(compiler.module_name, *tree[1:], **opts)
+            m_copy(module.__name__, *tree[1:], **opts)
         except TypeError as e:
             msg = "expanding `" + str(tree[0]) + "': "
             msg += str(e).replace("<lambda>()", "", 1).strip()
             raise HyMacroExpansionError(tree, msg)
 
         try:
-            obj = m(compiler.module_name, *tree[1:], **opts)
+            obj = m(module.__name__, *tree[1:], **opts)
         except HyTypeError as e:
             if e.expression is None:
                 e.expression = tree
@@ -225,25 +319,22 @@ def macroexpand(tree, compiler, once=False):
     tree = wrap_value(tree)
     return tree
 
-def macroexpand_1(tree, compiler):
+
+def macroexpand_1(tree, module, compiler=None):
     """Expand the toplevel macro from `tree` once, in the context of
     `compiler`."""
-    return macroexpand(tree, compiler, once=True)
+    return macroexpand(tree, module, compiler, once=True)
 
 
-def tag_macroexpand(tag, tree, compiler):
-    """Expand the tag macro "tag" with argument `tree`."""
-    load_macros(compiler.module_name)
+def tag_macroexpand(tag, tree, module):
+    """Expand the tag macro `tag` with argument `tree`."""
+    if not inspect.ismodule(module):
+        module = importlib.import_module(module)
+
+    tag_macro = module.__tags__.get(tag, None)
 
-    tag_macro = _hy_tag[compiler.module_name].get(tag)
     if tag_macro is None:
-        try:
-            tag_macro = _hy_tag[None][tag]
-        except KeyError:
-            raise HyTypeError(
-                tag,
-                "`{0}' is not a defined tag macro.".format(tag)
-            )
+        raise HyTypeError(tag, "'{0}' is not a defined tag macro.".format(tag))
 
     expr = tag_macro(tree)
     return replace_hy_obj(expr, tree)
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index 224670e..d33bfea 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -56,7 +56,7 @@ def test_runpy():
 
 
 def test_stringer():
-    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '')
+    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '__main__')
 
     assert type(_ast.body[0]) == ast.FunctionDef
 
@@ -80,7 +80,7 @@ def test_import_error_reporting():
 
     def _import_error_test():
         try:
-            _ = hy_compile(hy_parse("(import \"sys\")"), '')
+            _ = hy_compile(hy_parse("(import \"sys\")"), '__main__')
         except HyTypeError:
             return "Error reported"
 
diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py
index 407d756..8644532 100644
--- a/tests/macros/test_macro_processor.py
+++ b/tests/macros/test_macro_processor.py
@@ -22,6 +22,7 @@ def tmac(ETname, *tree):
 def test_preprocessor_simple():
     """ Test basic macro expansion """
     obj = macroexpand(tokenize('(test "one" "two")')[0],
+                      __name__,
                       HyASTCompiler(__name__))
     assert obj == HyList(["one", "two"])
     assert type(obj) == HyList
@@ -30,6 +31,7 @@ def test_preprocessor_simple():
 def test_preprocessor_expression():
     """ Test that macro expansion doesn't recurse"""
     obj = macroexpand(tokenize('(test (test "one" "two"))')[0],
+                      __name__,
                       HyASTCompiler(__name__))
 
     assert type(obj) == HyList
@@ -41,13 +43,13 @@ def test_preprocessor_expression():
 
     obj = HyList([HyString("one"), HyString("two")])
     obj = tokenize('(shill ["one" "two"])')[0][1]
-    assert obj == macroexpand(obj, HyASTCompiler(""))
+    assert obj == macroexpand(obj, __name__, HyASTCompiler(__name__))
 
 
 def test_preprocessor_exceptions():
     """ Test that macro expansion raises appropriate exceptions"""
     with pytest.raises(HyMacroExpansionError) as excinfo:
-        macroexpand(tokenize('(defn)')[0], HyASTCompiler(__name__))
+        macroexpand(tokenize('(defn)')[0], __name__, HyASTCompiler(__name__))
     assert "_hy_anon_fn_" not in excinfo.value.message
     assert "TypeError" not in excinfo.value.message
 
@@ -56,6 +58,6 @@ def test_macroexpand_nan():
    # https://github.com/hylang/hy/issues/1574
    import math
    NaN = float('nan')
-   x = macroexpand(HyFloat(NaN), HyASTCompiler(__name__))
+   x = macroexpand(HyFloat(NaN), __name__, HyASTCompiler(__name__))
    assert type(x) is HyFloat
    assert math.isnan(x)
diff --git a/tests/macros/test_tag_macros.py b/tests/macros/test_tag_macros.py
index 3cbfc94..f9c4f69 100644
--- a/tests/macros/test_tag_macros.py
+++ b/tests/macros/test_tag_macros.py
@@ -11,6 +11,7 @@ def test_tag_macro_error():
     """Check if we get correct error with wrong dispatch character"""
     try:
         macroexpand(tokenize("(dispatch_tag_macro '- '())")[0],
+                    __name__,
                     HyASTCompiler(__name__))
     except HyTypeError as e:
         assert "with the character `-`" in str(e)
diff --git a/tests/resources/bin/circular_macro_require.hy b/tests/resources/bin/circular_macro_require.hy
new file mode 100644
index 0000000..62d0ce2
--- /dev/null
+++ b/tests/resources/bin/circular_macro_require.hy
@@ -0,0 +1,8 @@
+(defmacro bar [expr]
+  `(print ~expr))
+
+(defmacro foo [expr]
+  `(do (require [tests.resources.bin.circular-macro-require [bar]])
+       (bar ~expr)))
+
+(foo 42)
diff --git a/tests/resources/bin/require_and_eval.hy b/tests/resources/bin/require_and_eval.hy
new file mode 100644
index 0000000..4e1c144
--- /dev/null
+++ b/tests/resources/bin/require_and_eval.hy
@@ -0,0 +1,3 @@
+(require [hy.extra.anaphoric [ap-if]])
+
+(print (eval '(ap-if (+ "a" "b") (+ it "c"))))
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 58d1448..6d1e10e 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -388,3 +388,38 @@ def test_bin_hy_file_no_extension():
     """Confirm that a file with no extension is processed as Hy source"""
     output, _ = run_cmd("hy tests/resources/no_extension")
     assert "This Should Still Work" in output
+
+
+def test_bin_hy_circular_macro_require():
+    """Confirm that macros can require themselves during expansion and when
+    run from the command line."""
+
+    # First, with no bytecode
+    test_file = "tests/resources/bin/circular_macro_require.hy"
+    rm(cache_from_source(test_file))
+    assert not os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "42" == output.strip()
+
+    # Now, with bytecode
+    assert os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "42" == output.strip()
+
+def test_bin_hy_macro_require():
+    """Confirm that a `require` will load macros into the non-module namespace
+    (i.e. `exec(code, locals)`) used by `runpy.run_path`.
+    In other words, this confirms that the AST generated for a `require` will
+    load macros into the unnamed namespace its run in."""
+
+    # First, with no bytecode
+    test_file = "tests/resources/bin/require_and_eval.hy"
+    rm(cache_from_source(test_file))
+    assert not os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "abc" == output.strip()
+
+    # Now, with bytecode
+    assert os.path.exists(cache_from_source(test_file))
+    output, _ = run_cmd("hy {}".format(test_file))
+    assert "abc" == output.strip()

From 010986e8cabf7b3b308ffc0cd679cc8e6d8e036a Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Tue, 23 Oct 2018 14:06:22 -0400
Subject: [PATCH 079/223] Implement minimal macro namespacing and add tests

This commit adds just enough namespacing to resolve a macro first in the macro's
defining module's namespace (i.e. the module assigned to the `HyASTCompiler`),
then in the namespace/module it's evaluated in.  Namespacing is accomplished by
adding a `module` attribute to `HySymbol`, so that `HyExpression`s can be
checked for this definition namespace attribute and their car symbol resolved
per the above.

As well, a couple tests have been added that cover
- the loading of module-level macros
  - e.g. that only macros defined in the `require`d module are added
- the AST generated for `require`
  - using macros loaded from modules imported via bytecode
- the non-local macro namespace resolution described above
  - a `require`d macro that uses a macro `require` exclusively in its
    module-level namespace
- and that (second-degree `require`d) macros can reference variables within
  their module-level namespaces.

Closes hylang/hy#1268, closes hylang/hy#1650, closes hylang/hy#1416.
---
 hy/macros.py                          |  60 +++++++++++--
 hy/models.py                          |   5 +-
 tests/native_tests/native_macros.hy   | 124 +++++++++++++++++++++++++-
 tests/resources/macro_with_require.hy |  25 ++++++
 tests/resources/macros.hy             |  10 +++
 5 files changed, 216 insertions(+), 8 deletions(-)
 create mode 100644 tests/resources/macro_with_require.hy

diff --git a/hy/macros.py b/hy/macros.py
index f3b64ff..0a6bc1b 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -263,10 +263,38 @@ def make_empty_fn_copy(fn):
 
 
 def macroexpand(tree, module, compiler=None, once=False):
-    """Expand the toplevel macros for the `tree`.
+    """Expand the toplevel macros for the given Hy AST tree.
 
-    Load the macros from the given `module`, then expand the
-    (top-level) macros in `tree` until we no longer can.
+    Load the macros from the given `module`, then expand the (top-level) macros
+    in `tree` until we no longer can.
+
+    `HyExpression` resulting from macro expansions are assigned the module in
+    which the macro function is defined (determined using `inspect.getmodule`).
+    If the resulting `HyExpression` is itself macro expanded, then the
+    namespace of the assigned module is checked first for a macro corresponding
+    to the expression's head/car symbol.  If the head/car symbol of such a
+    `HyExpression` is not found among the macros of its assigned module's
+    namespace, the outer-most namespace--e.g.  the one given by the `module`
+    parameter--is used as a fallback.
+
+    Parameters
+    ----------
+    tree: HyObject or list
+        Hy AST tree.
+
+    module: str or types.ModuleType
+        Module used to determine the local namespace for macros.
+
+    compiler: HyASTCompiler, optional
+        The compiler object passed to expanded macros.
+
+    once: boolean, optional
+        Only expand the first macro in `tree`.
+
+    Returns
+    ------
+    out: HyObject
+        Returns a mutated tree with macros expanded.
     """
     if not inspect.ismodule(module):
         module = importlib.import_module(module)
@@ -283,7 +311,14 @@ def macroexpand(tree, module, compiler=None, once=False):
             break
 
         fn = mangle(fn)
-        m = module.__macros__.get(fn, None)
+        expr_modules = (([] if not hasattr(tree, 'module') else [tree.module])
+            + [module])
+
+        # Choose the first namespace with the macro.
+        m = next((mod.__macros__[fn]
+                  for mod in expr_modules
+                  if fn in mod.__macros__),
+                 None)
         if not m:
             break
 
@@ -311,6 +346,10 @@ def macroexpand(tree, module, compiler=None, once=False):
         except Exception as e:
             msg = "expanding `" + str(tree[0]) + "': " + repr(e)
             raise HyMacroExpansionError(tree, msg)
+
+        if isinstance(obj, HyExpression):
+            obj.module = inspect.getmodule(m)
+
         tree = replace_hy_obj(obj, tree)
 
         if once:
@@ -331,10 +370,21 @@ def tag_macroexpand(tag, tree, module):
     if not inspect.ismodule(module):
         module = importlib.import_module(module)
 
-    tag_macro = module.__tags__.get(tag, None)
+    expr_modules = (([] if not hasattr(tree, 'module') else [tree.module])
+        + [module])
+
+    # Choose the first namespace with the macro.
+    tag_macro = next((mod.__tags__[tag]
+                      for mod in expr_modules
+                      if tag in mod.__tags__),
+                     None)
 
     if tag_macro is None:
         raise HyTypeError(tag, "'{0}' is not a defined tag macro.".format(tag))
 
     expr = tag_macro(tree)
+
+    if isinstance(expr, HyExpression):
+        expr.module = inspect.getmodule(tag_macro)
+
     return replace_hy_obj(expr, tree)
diff --git a/hy/models.py b/hy/models.py
index 943458e..2ff5efa 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -32,11 +32,12 @@ class HyObject(object):
     Generic Hy Object model. This is helpful to inject things into all the
     Hy lexing Objects at once.
     """
+    __properties__ = ["module", "start_line", "end_line", "start_column",
+                      "end_column"]
 
     def replace(self, other, recursive=False):
         if isinstance(other, HyObject):
-            for attr in ["start_line", "end_line",
-                         "start_column", "end_column"]:
+            for attr in self.__properties__:
                 if not hasattr(self, attr) and hasattr(other, attr):
                     setattr(self, attr, getattr(other, attr))
         else:
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index ef6ae26..73b433b 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -2,7 +2,8 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import [hy.errors [HyTypeError]])
+(import pytest
+        [hy.errors [HyTypeError]])
 
 (defmacro rev [&rest body]
   "Execute the `body` statements in reverse"
@@ -329,3 +330,124 @@
     (except [e SystemExit]
       (assert (= (str e) "42"))))
   (setv --name-- oldname))
+
+(defn test-macro-namespace-resolution []
+  "Confirm that local versions of macro-macro dependencies do not shadow the
+versions from the macro's own module, but do resolve unbound macro references
+in expansions."
+
+  ;; `nonlocal-test-macro` is a macro used within
+  ;; `tests.resources.macro-with-require.test-module-macro`.
+  ;; Here, we introduce an equivalently named version in local scope that, when
+  ;; used, will expand to a different output string.
+  (defmacro nonlocal-test-macro [x]
+    (print "this is the local version of `nonlocal-test-macro`!"))
+
+  ;; Was the above macro created properly?
+  (assert (in "nonlocal_test_macro" __macros__))
+
+  (setv nonlocal-test-macro (get __macros__ "nonlocal_test_macro"))
+
+  (require [tests.resources.macro-with-require [*]])
+
+  ;; Make sure our local version wasn't overwritten by a faulty `require` of the
+  ;; one in tests.resources.macro-with-require.
+  (assert (= nonlocal-test-macro (get __macros__ "nonlocal_test_macro")))
+
+  (setv module-name-var "tests.native_tests.native_macros.test-macro-namespace-resolution")
+  (assert (= (+ "This macro was created in tests.resources.macros, "
+                "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution "
+                "and passed the value 2.")
+             (test-module-macro 2)))
+  (assert (= (+ "This macro was created in tests.resources.macros, "
+                "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution "
+                "and passed the value 2.")
+             #test-module-tag 2))
+
+  ;; Now, let's use a `require`d macro that depends on another macro defined only
+  ;; in this scope.
+  (defmacro local-test-macro [x]
+    (.format "This is the local version of `nonlocal-test-macro` returning {}!" x))
+
+  (assert (= "This is the local version of `nonlocal-test-macro` returning 3!"
+             (test-module-macro-2 3)))
+  (assert (= "This is the local version of `nonlocal-test-macro` returning 3!"
+             #test-module-tag-2 3)))
+
+(defn test-macro-from-module []
+  "Macros loaded from an external module, which itself `require`s macros, should
+ work without having to `require` the module's macro dependencies (due to
+ [minimal] macro namespace resolution).
+
+ In doing so we also confirm that a module's `__macros__` attribute is correctly
+ loaded and used.
+
+ Additionally, we confirm that `require` statements are executed via loaded bytecode."
+
+  (import os sys marshal types)
+  (import [hy.importer [cache-from-source]])
+
+  (setv pyc-file (cache-from-source
+                   (os.path.realpath
+                     (os.path.join
+                       "tests" "resources" "macro_with_require.hy"))))
+
+  ;; Remove any cached byte-code, so that this runs from source and
+  ;; gets evaluated in this module.
+  (when (os.path.isfile pyc-file)
+    (os.unlink pyc-file)
+    (.clear sys.path_importer_cache)
+    (when (in  "tests.resources.macro_with_require" sys.modules)
+      (del (get sys.modules "tests.resources.macro_with_require"))
+      (__macros__.clear)
+      (__tags__.clear)))
+
+  ;; Ensure that bytecode isn't present when we require this module.
+  (assert (not (os.path.isfile pyc-file)))
+
+  (defn test-requires-and-macros []
+    (require [tests.resources.macro-with-require
+              [test-module-macro]])
+
+    ;; Make sure that `require` didn't add any of its `require`s
+    (assert (not (in "nonlocal-test-macro" __macros__)))
+    ;; and that it didn't add its tags.
+    (assert (not (in "test_module_tag" __tags__)))
+
+    ;; Now, require everything.
+    (require [tests.resources.macro-with-require [*]])
+
+    ;; Again, make sure it didn't add its required macros and/or tags.
+    (assert (not (in "nonlocal-test-macro" __macros__)))
+
+    ;; Its tag(s) should be here now.
+    (assert (in "test_module_tag" __tags__))
+
+    ;; The test macro expands to include this symbol.
+    (setv module-name-var "tests.native_tests.native_macros")
+    (assert (= (+ "This macro was created in tests.resources.macros, "
+                  "expanded in tests.native_tests.native_macros "
+                  "and passed the value 1.")
+               (test-module-macro 1)))
+
+    (assert (= (+ "This macro was created in tests.resources.macros, "
+                  "expanded in tests.native_tests.native_macros "
+                  "and passed the value 1.")
+               #test-module-tag 1)))
+
+  (test-requires-and-macros)
+
+  ;; Now that bytecode is present, reload the module, clear the `require`d
+  ;; macros and tags, and rerun the tests.
+  (assert (os.path.isfile pyc-file))
+
+  ;; Reload the module and clear the local macro context.
+  (.clear sys.path_importer_cache)
+  (del (get sys.modules "tests.resources.macro_with_require"))
+  (.clear __macros__)
+  (.clear __tags__)
+
+  ;; XXX: There doesn't seem to be a way--via standard import mechanisms--to
+  ;; ensure that an imported module used the cached bytecode.  We'll simply have
+  ;; to trust that the .pyc loading convention was followed.
+  (test-requires-and-macros))
diff --git a/tests/resources/macro_with_require.hy b/tests/resources/macro_with_require.hy
new file mode 100644
index 0000000..ef25018
--- /dev/null
+++ b/tests/resources/macro_with_require.hy
@@ -0,0 +1,25 @@
+;; Require all the macros and make sure they don't pollute namespaces/modules
+;; that require `*` from this.
+(require [tests.resources.macros [*]])
+
+(defmacro test-module-macro [a]
+  "The variable `macro-level-var' here should not bind to the same-named symbol
+in the expansion of `nonlocal-test-macro'."
+  (setv macro-level-var "tests.resources.macros.macro-with-require")
+  `(nonlocal-test-macro ~a))
+
+(deftag test-module-tag [a]
+  "The variable `macro-level-var' here should not bind to the same-named symbol
+in the expansion of `nonlocal-test-macro'."
+  (setv macro-level-var "tests.resources.macros.macro-with-require")
+  `(nonlocal-test-macro ~a))
+
+(defmacro test-module-macro-2 [a]
+  "The macro `local-test-macro` isn't in this module's namespace, so it better
+ be in the expansion's!"
+  `(local-test-macro ~a))
+
+(deftag test-module-tag-2 [a]
+  "The macro `local-test-macro` isn't in this module's namespace, so it better
+ be in the expansion's!"
+  `(local-test-macro ~a))
diff --git a/tests/resources/macros.hy b/tests/resources/macros.hy
index 300a790..4a17a05 100644
--- a/tests/resources/macros.hy
+++ b/tests/resources/macros.hy
@@ -1,3 +1,5 @@
+(setv module-name-var "tests.resources.macros")
+
 (defmacro thread-set-ab []
   (defn f [&rest args] (.join "" (+ (, "a") args)))
   (setv variable (HySymbol (-> "b" (f))))
@@ -10,3 +12,11 @@
 
 (defmacro test-macro []
   '(setv blah 1))
+
+(defmacro nonlocal-test-macro [x]
+  "When called from `macro-with-require`'s macro(s), the first instance of
+`module-name-var` should resolve to the value in the module where this is
+defined, then the expansion namespace/module"
+  `(.format (+ "This macro was created in {}, expanded in {} "
+               "and passed the value {}.")
+            ~module-name-var module-name-var ~x))
\ No newline at end of file

From 3b01742818ef681031452c3f46496097d79f5d1f Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sun, 14 Oct 2018 01:04:41 -0500
Subject: [PATCH 080/223] Update NEWS

---
 NEWS.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 5a67e80..bb101a2 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -12,9 +12,15 @@ New Features
 * Keyword objects (not just literal keywords) can be called, as
   shorthand for `(get obj :key)`, and they accept a default value
   as a second argument.
+* Minimal macro expansion namespacing has been implemented.  As a result,
+  external macros no longer have to `require` their own macro dependencies.
+* Macros and tags now reside in module-level `__macros__` and `__tags__`
+  attributes.
 
 Bug Fixes
 ------------------------------
+* `require` now compiles to Python AST.
+* Fixed circular `require`s.
 * Fixed module reloading.
 * Fixed circular imports.
 * Fixed `__main__` file execution.

From aa9182d76c3999a8682b1f86dc13050046bbf810 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 24 Oct 2018 15:47:19 -0500
Subject: [PATCH 081/223] Make the stdlib dictionary a class instance variable

---
 hy/compiler.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index ceef84b..f6cf052 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -281,9 +281,6 @@ def is_unpack(kind, x):
             and x[0] == "unpack-" + kind)
 
 
-_stdlib = {}
-
-
 class HyASTCompiler(object):
     """A Hy-to-Python AST compiler"""
 
@@ -311,9 +308,11 @@ class HyASTCompiler(object):
         # Load stdlib macros into the module namespace.
         load_macros(self.module)
 
+        self._stdlib = {}
+
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
-        if self.can_use_stdlib and not _stdlib:
+        if self.can_use_stdlib:
             # Populate _stdlib.
             import hy.core
             for stdlib_module in hy.core.STDLIB:
@@ -323,7 +322,7 @@ class HyASTCompiler(object):
                         # Don't bother putting a name in _stdlib if it
                         # points to a builtin with the same name. This
                         # prevents pointless imports.
-                        _stdlib[e] = stdlib_module
+                        self._stdlib[e] = stdlib_module
 
     def get_anon_var(self):
         self.anon_var_count += 1
@@ -1690,8 +1689,8 @@ class HyASTCompiler(object):
                 attr=ast_str(local),
                 ctx=ast.Load())
 
-        if self.can_use_stdlib and ast_str(symbol) in _stdlib:
-            self.imports[_stdlib[ast_str(symbol)]].add(ast_str(symbol))
+        if self.can_use_stdlib and ast_str(symbol) in self._stdlib:
+            self.imports[self._stdlib[ast_str(symbol)]].add(ast_str(symbol))
 
         return asty.Name(symbol, id=ast_str(symbol), ctx=ast.Load())
 

From 28504ba85da698884d7e4bb804c14ad492a2ff3b Mon Sep 17 00:00:00 2001
From: Jakub Wilk <jwilk@jwilk.net>
Date: Wed, 7 Nov 2018 20:15:43 +0100
Subject: [PATCH 082/223] Catch IndentationError in isidentifier()

Fixes:

    >>> from hy._compat import isidentifier
    >>> isidentifier(u"  0\n 0")
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
      File "hy/_compat.py", line 47, in isidentifier
        tokens = list(T.generate_tokens(StringIO(x).readline))
      File "/usr/lib/python2.7/tokenize.py", line 374, in generate_tokens
        ("<tokenize>", lnum, pos, line))
      File "<tokenize>", line 2
        0
        ^
    IndentationError: unindent does not match any outer indentation level
---
 NEWS.rst                       | 1 +
 hy/_compat.py                  | 2 +-
 tests/native_tests/mangling.hy | 4 ++++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/NEWS.rst b/NEWS.rst
index bb101a2..44ab3cb 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -30,6 +30,7 @@ Bug Fixes
   objects.
 * Fixed errors from `from __future__ import ...` statements and missing Hy
   module docstrings caused by automatic importing of Hy builtins.
+* Fixed crash in `mangle` for some pathological inputs
 
 0.15.0
 ==============================
diff --git a/hy/_compat.py b/hy/_compat.py
index 4416ea5..379ce43 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -45,7 +45,7 @@ def isidentifier(x):
     from io import StringIO
     try:
         tokens = list(T.generate_tokens(StringIO(x).readline))
-    except T.TokenError:
+    except (T.TokenError, IndentationError):
         return False
     return len(tokens) == 2 and tokens[0][0] == T.NAME
 
diff --git a/tests/native_tests/mangling.hy b/tests/native_tests/mangling.hy
index e009ab6..bfb4d0a 100644
--- a/tests/native_tests/mangling.hy
+++ b/tests/native_tests/mangling.hy
@@ -157,3 +157,7 @@
                ["⚘-⚘" "hyx_XflowerX_XflowerX"]]]
     (assert (= (mangle a) b))
     (assert (= (unmangle b) a))))
+
+(defn test-mangle-bad-indent []
+  ; Shouldn't crash with IndentationError
+  (mangle "  0\n 0"))

From 86fda31ab1d559e08b0852ae7b78973be52b80d3 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sat, 10 Nov 2018 12:53:28 -0600
Subject: [PATCH 083/223] Move compilation and parsing functions out of
 `importer.py`

Functions and variables relating to compilation and parsing have been moved to
`compiler.py` and `lex/__init__.py`, respectively.  Those functions are
  - `hy_parse` from `hy.importer` to `hy.lex`
  - `hy_eval`, `ast_compile`, and `calling_module` from `hy.importer` to
  `hy.compiler`

Closes hylang/hy#1695.
---
 hy/__init__.py                      |   2 +-
 hy/cmdline.py                       |   7 +-
 hy/compiler.py                      | 137 +++++++++++++++++++++++-
 hy/core/language.hy                 |   6 +-
 hy/importer.py                      | 157 +---------------------------
 hy/lex/__init__.py                  |  24 ++++-
 tests/compilers/test_ast.py         |   3 +-
 tests/importer/test_importer.py     |   5 +-
 tests/native_tests/native_macros.hy |  12 ++-
 tests/test_lex.py                   |   3 +-
 10 files changed, 182 insertions(+), 174 deletions(-)

diff --git a/hy/__init__.py b/hy/__init__.py
index d13c35a..30248e0 100644
--- a/hy/__init__.py
+++ b/hy/__init__.py
@@ -13,4 +13,4 @@ import hy.importer  # NOQA
 
 
 from hy.core.language import read, read_str, mangle, unmangle  # NOQA
-from hy.importer import hy_eval as eval  # NOQA
+from hy.compiler import hy_eval as eval  # NOQA
diff --git a/hy/cmdline.py b/hy/cmdline.py
index eeb5ccd..7fdfb49 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -18,9 +18,10 @@ import types
 import astor.code_gen
 
 import hy
-from hy.lex import LexException, PrematureEndOfInput, mangle
-from hy.compiler import HyTypeError, hy_compile
-from hy.importer import hy_eval, hy_parse, runhy
+from hy.lex import hy_parse, mangle
+from hy.lex.exceptions import LexException, PrematureEndOfInput
+from hy.compiler import HyTypeError, hy_compile, hy_eval
+from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
diff --git a/hy/compiler.py b/hy/compiler.py
index f6cf052..2b262a1 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -16,7 +16,6 @@ from hy.lex import mangle, unmangle
 from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
                         PY35, raise_empty)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
-import hy.importer
 
 import traceback
 import importlib
@@ -26,6 +25,7 @@ import types
 import ast
 import sys
 import copy
+import __future__
 
 from collections import defaultdict
 
@@ -37,6 +37,60 @@ else:
 Inf = float('inf')
 
 
+hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
+                        __future__.CO_FUTURE_PRINT_FUNCTION)
+
+
+def ast_compile(ast, filename, mode):
+    """Compile AST.
+
+    Parameters
+    ----------
+    ast : instance of `ast.AST`
+
+    filename : str
+        Filename used for run-time error messages
+
+    mode: str
+        `compile` mode parameter
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    return compile(ast, filename, mode, hy_ast_compile_flags)
+
+
+def calling_module(n=1):
+    """Get the module calling, if available.
+
+    As a fallback, this will import a module using the calling frame's
+    globals value of `__name__`.
+
+    Parameters
+    ----------
+    n: int, optional
+        The number of levels up the stack from this function call.
+        The default is one level up.
+
+    Returns
+    -------
+    out: types.ModuleType
+        The module at stack level `n + 1` or `None`.
+    """
+    frame_up = inspect.stack(0)[n + 1][0]
+    module = inspect.getmodule(frame_up)
+    if module is None:
+        # This works for modules like `__main__`
+        module_name = frame_up.f_globals.get('__name__', None)
+        if module_name:
+            try:
+                module = importlib.import_module(module_name)
+            except ImportError:
+                pass
+    return module
+
+
 def ast_str(x, piecewise=False):
     if piecewise:
         return ".".join(ast_str(s) if s else "" for s in x.split("."))
@@ -1554,9 +1608,7 @@ class HyASTCompiler(object):
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
 
-        hy.importer.hy_eval(new_expr + body,
-                            self.module.__dict__,
-                            self.module)
+        hy_eval(new_expr + body, self.module.__dict__, self.module)
 
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
@@ -1723,6 +1775,83 @@ class HyASTCompiler(object):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
+def hy_eval(hytree, locals=None, module=None, ast_callback=None):
+    """Evaluates a quoted expression and returns the value.
+
+    Examples
+    --------
+
+       => (eval '(print "Hello World"))
+       "Hello World"
+
+    If you want to evaluate a string, use ``read-str`` to convert it to a
+    form first:
+
+       => (eval (read-str "(+ 1 1)"))
+       2
+
+    Parameters
+    ----------
+    hytree: a Hy expression tree
+        Source code to parse.
+
+    locals: dict, optional
+        Local environment in which to evaluate the Hy tree.  Defaults to the
+        calling frame.
+
+    module: str or types.ModuleType, optional
+        Module, or name of the module, to which the Hy tree is assigned and
+        the global values are taken.
+        Defaults to the calling frame's module, if any, and '__eval__'
+        otherwise.
+
+    ast_callback: callable, optional
+        A callback that is passed the Hy compiled tree and resulting
+        expression object, in that order, after compilation but before
+        evaluation.
+
+    Returns
+    -------
+    out : Result of evaluating the Hy compiled tree.
+    """
+    if module is None:
+        module = calling_module()
+
+    if isinstance(module, string_types):
+        module = importlib.import_module(ast_str(module, piecewise=True))
+    elif not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    if locals is None:
+        frame = inspect.stack()[1][0]
+        locals = inspect.getargvalues(frame).locals
+
+    if not isinstance(locals, dict):
+        raise TypeError("Locals must be a dictionary")
+
+    _ast, expr = hy_compile(hytree, module, get_expr=True)
+
+    # Spoof the positions in the generated ast...
+    for node in ast.walk(_ast):
+        node.lineno = 1
+        node.col_offset = 1
+
+    for node in ast.walk(expr):
+        node.lineno = 1
+        node.col_offset = 1
+
+    if ast_callback:
+        ast_callback(_ast, expr)
+
+    globals = module.__dict__
+
+    # Two-step eval: eval() the body of the exec call
+    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
+
+    # Then eval the expression context and return that
+    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
+
+
 def hy_compile(tree, module, root=ast.Module, get_expr=False):
     """
     Compile a Hy tree into a Python AST tree.
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 5ada235..981b3d3 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -19,9 +19,9 @@
   (import [collections :as cabc])
   (import [collections.abc :as cabc]))
 (import [hy.models [HySymbol HyKeyword]])
-(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]])
-(import [hy.compiler [HyASTCompiler]])
-(import [hy.importer [calling-module hy-eval :as eval]])
+(import [hy.lex [tokenize mangle unmangle]])
+(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
+(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
diff --git a/hy/importer.py b/hy/importer.py
index fdcb0be..5e68c61 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import sys
 import os
-import ast
 import inspect
 import pkgutil
 import re
@@ -14,163 +13,15 @@ import io
 import types
 import tempfile
 import importlib
-import __future__
 
 from functools import partial
 from contextlib import contextmanager
 
 from hy.errors import HyTypeError
-from hy.compiler import hy_compile, ast_str
-from hy.lex import tokenize, LexException
-from hy.models import HyExpression, HySymbol
-from hy._compat import string_types, PY3
-
-
-hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
-                        __future__.CO_FUTURE_PRINT_FUNCTION)
-
-
-def calling_module(n=1):
-    """Get the module calling, if available.
-
-    As a fallback, this will import a module using the calling frame's
-    globals value of `__name__`.
-
-    Parameters
-    ----------
-    n: int, optional
-        The number of levels up the stack from this function call.
-        The default is one level up.
-
-    Returns
-    -------
-    out: types.ModuleType
-        The module at stack level `n + 1` or `None`.
-    """
-    frame_up = inspect.stack(0)[n + 1][0]
-    module = inspect.getmodule(frame_up)
-    if module is None:
-        # This works for modules like `__main__`
-        module_name = frame_up.f_globals.get('__name__', None)
-        if module_name:
-            try:
-                module = importlib.import_module(module_name)
-            except ImportError:
-                pass
-    return module
-
-
-def ast_compile(ast, filename, mode):
-    """Compile AST.
-
-    Parameters
-    ----------
-    ast : instance of `ast.AST`
-
-    filename : str
-        Filename used for run-time error messages
-
-    mode: str
-        `compile` mode parameter
-
-    Returns
-    -------
-    out : instance of `types.CodeType`
-    """
-    return compile(ast, filename, mode, hy_ast_compile_flags)
-
-
-def hy_parse(source):
-    """Parse a Hy source string.
-
-    Parameters
-    ----------
-    source: string
-        Source code to parse.
-
-    Returns
-    -------
-    out : instance of `types.CodeType`
-    """
-    source = re.sub(r'\A#!.*', '', source)
-    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
-
-
-def hy_eval(hytree, locals=None, module=None, ast_callback=None):
-    """Evaluates a quoted expression and returns the value.
-
-    Examples
-    --------
-
-       => (eval '(print "Hello World"))
-       "Hello World"
-
-    If you want to evaluate a string, use ``read-str`` to convert it to a
-    form first:
-
-       => (eval (read-str "(+ 1 1)"))
-       2
-
-    Parameters
-    ----------
-    hytree: a Hy expression tree
-        Source code to parse.
-
-    locals: dict, optional
-        Local environment in which to evaluate the Hy tree.  Defaults to the
-        calling frame.
-
-    module: str or types.ModuleType, optional
-        Module, or name of the module, to which the Hy tree is assigned and
-        the global values are taken.
-        Defaults to the calling frame's module, if any, and '__eval__'
-        otherwise.
-
-    ast_callback: callable, optional
-        A callback that is passed the Hy compiled tree and resulting
-        expression object, in that order, after compilation but before
-        evaluation.
-
-    Returns
-    -------
-    out : Result of evaluating the Hy compiled tree.
-    """
-    if module is None:
-        module = calling_module()
-
-    if isinstance(module, string_types):
-        module = importlib.import_module(ast_str(module, piecewise=True))
-    elif not inspect.ismodule(module):
-        raise TypeError('Invalid module type: {}'.format(type(module)))
-
-    if locals is None:
-        frame = inspect.stack()[1][0]
-        locals = inspect.getargvalues(frame).locals
-
-    if not isinstance(locals, dict):
-        raise TypeError("Locals must be a dictionary")
-
-    _ast, expr = hy_compile(hytree, module, get_expr=True)
-
-    # Spoof the positions in the generated ast...
-    for node in ast.walk(_ast):
-        node.lineno = 1
-        node.col_offset = 1
-
-    for node in ast.walk(expr):
-        node.lineno = 1
-        node.col_offset = 1
-
-    if ast_callback:
-        ast_callback(_ast, expr)
-
-    globals = module.__dict__
-
-    # Two-step eval: eval() the body of the exec call
-    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
-
-    # Then eval the expression context and return that
-    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
+from hy.compiler import hy_compile, hy_ast_compile_flags
+from hy.lex import hy_parse
+from hy.lex.exceptions import LexException
+from hy._compat import PY3
 
 
 def cache_from_source(source_path):
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index 5c05143..093fa49 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -4,9 +4,29 @@
 
 from __future__ import unicode_literals
 
-import re, unicodedata
+import re
+import unicodedata
+
 from hy._compat import str_type, isidentifier, UCS4
-from hy.lex.exceptions import LexException, PrematureEndOfInput  # NOQA
+from hy.lex.exceptions import LexException  # NOQA
+from hy.models import HyExpression, HySymbol
+
+
+def hy_parse(source):
+    """Parse a Hy source string.
+
+    Parameters
+    ----------
+    source: string
+        Source code to parse.
+
+    Returns
+    -------
+    out : instance of `types.CodeType`
+    """
+    source = re.sub(r'\A#!.*', '', source)
+    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
+
 
 def tokenize(buf):
     """
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index cb3ab7b..1f94c3f 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -7,8 +7,9 @@ from __future__ import unicode_literals
 
 from hy import HyString
 from hy.models import HyObject
-from hy.importer import hy_compile, hy_eval, hy_parse
+from hy.compiler import hy_compile, hy_eval
 from hy.errors import HyCompileError, HyTypeError
+from hy.lex import hy_parse
 from hy.lex.exceptions import LexException
 from hy._compat import PY3
 
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index d33bfea..1973fe8 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -15,9 +15,10 @@ import pytest
 
 import hy
 from hy.errors import HyTypeError
-from hy.lex import LexException
+from hy.lex import hy_parse
+from hy.lex.exceptions import LexException
 from hy.compiler import hy_compile
-from hy.importer import hy_parse, HyLoader, cache_from_source
+from hy.importer import HyLoader, cache_from_source
 
 try:
     from importlib import reload
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 73b433b..fc17edb 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -148,7 +148,8 @@
 (defn test-gensym-in-macros []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (setv g (gensym))
       `(do
@@ -174,7 +175,8 @@
 (defn test-with-gensym []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro nif [expr pos zero neg]
       (with-gensyms [a]
         `(do
@@ -198,7 +200,8 @@
 (defn test-defmacro/g! []
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro/g! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
@@ -227,7 +230,8 @@
   ;; defmacro! must do everything defmacro/g! can
   (import ast)
   (import [astor.code-gen [to-source]])
-  (import [hy.importer [hy-parse hy-compile]])
+  (import [hy.compiler [hy-compile]])
+  (import [hy.lex [hy-parse]])
   (setv macro1 "(defmacro! nif [expr pos zero neg]
         `(do
            (setv ~g!res ~expr)
diff --git a/tests/test_lex.py b/tests/test_lex.py
index 411c246..2e68876 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -5,7 +5,8 @@
 from math import isnan
 from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol,
                        HyString, HyDict, HyList, HySet, HyKeyword)
-from hy.lex import LexException, PrematureEndOfInput, tokenize
+from hy.lex import tokenize
+from hy.lex.exceptions import LexException, PrematureEndOfInput
 import pytest
 
 def peoi(): return pytest.raises(PrematureEndOfInput)

From 8b6646d5c9800c7bfb62e5a26d0b808786cd2bca Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sat, 10 Nov 2018 16:19:41 -0600
Subject: [PATCH 084/223] Remove `hy.core` compilation requirement from `hy`
 package

Previously, when importing `hy` (and any of its sub-packages/modules), Hy source
compilation for `hy.core.language` was necessarily triggered.  This, in turn,
would trigger compilation of the other standard library source files.

This commit removes that chain of events and allows the `hy` package to be
imported without any Hy compilation.

Furthermore, `read` and `read_str` are now implemented in Python and the Hy
standard library files now handle their own dependencies explicitly (i.e. they
`import` and/or `require` the other standard library files upon which they
depend).

The latter changes were necessary, because the automatically triggered
compilation of `hy.core.language` (and associated standard library files) was
serving--implicitly--as a means of producing bytecode in an order that just
happened to work for any compilation occurring afterward.  This chain of
events/dependencies was extremely cryptic, brittle, and difficult to debug, and
these changes should help to remedy that.

Closes hylang/hy#1697.
---
 hy/__init__.py      |  2 +-
 hy/compiler.py      | 16 ++++++++++------
 hy/core/language.hy | 38 +++++++++-----------------------------
 hy/core/macros.hy   |  5 +++++
 hy/core/shadow.hy   |  2 ++
 hy/lex/__init__.py  | 33 ++++++++++++++++++++++++++++++++-
 6 files changed, 59 insertions(+), 37 deletions(-)

diff --git a/hy/__init__.py b/hy/__init__.py
index 30248e0..f188b64 100644
--- a/hy/__init__.py
+++ b/hy/__init__.py
@@ -12,5 +12,5 @@ import hy.importer  # NOQA
 # we import for side-effects.
 
 
-from hy.core.language import read, read_str, mangle, unmangle  # NOQA
+from hy.lex import read, read_str, mangle, unmangle  # NOQA
 from hy.compiler import hy_eval as eval  # NOQA
diff --git a/hy/compiler.py b/hy/compiler.py
index 2b262a1..11473d7 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -17,6 +17,8 @@ from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
                         PY35, raise_empty)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
+import hy.core
+
 import traceback
 import importlib
 import inspect
@@ -355,20 +357,22 @@ class HyASTCompiler(object):
         self.module = module
         self.module_name = module.__name__
 
-        self.can_use_stdlib = (
-            not self.module_name.startswith("hy.core")
-            or self.module_name == "hy.core.macros")
+        # Hy expects these to be present, so we prep the module for Hy
+        # compilation.
+        self.module.__dict__.setdefault('__macros__', {})
+        self.module.__dict__.setdefault('__tags__', {})
 
-        # Load stdlib macros into the module namespace.
-        load_macros(self.module)
+        self.can_use_stdlib = not self.module_name.startswith("hy.core")
 
         self._stdlib = {}
 
         # Everything in core needs to be explicit (except for
         # the core macros, which are built with the core functions).
         if self.can_use_stdlib:
+            # Load stdlib macros into the module namespace.
+            load_macros(self.module)
+
             # Populate _stdlib.
-            import hy.core
             for stdlib_module in hy.core.STDLIB:
                 mod = importlib.import_module(stdlib_module)
                 for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 981b3d3..e13cbfa 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -11,17 +11,19 @@
 (import [fractions [Fraction :as fraction]])
 (import operator)  ; shadow not available yet
 (import sys)
-(if-python2
-  (import [StringIO [StringIO]])
-  (import [io [StringIO]]))
 (import [hy._compat [long-type]]) ; long for python2, int for python3
+(import [hy.models [HySymbol HyKeyword]])
+(import [hy.lex [tokenize mangle unmangle read read-str]])
+(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
+(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
+
+(import [hy.core.shadow [*]])
+
+(require [hy.core.bootstrap [*]])
+
 (if-python2
   (import [collections :as cabc])
   (import [collections.abc :as cabc]))
-(import [hy.models [HySymbol HyKeyword]])
-(import [hy.lex [tokenize mangle unmangle]])
-(import [hy.lex.exceptions [LexException PrematureEndOfInput]])
-(import [hy.compiler [HyASTCompiler calling-module hy-eval :as eval]])
 
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
@@ -415,28 +417,6 @@ Raises ValueError for (not (pos? n))."
   "Check if `n` equals 0."
   (= n 0))
 
-(defn read [&optional [from-file sys.stdin]
-                      [eof ""]]
-  "Read from input and returns a tokenized string.
-
-Can take a given input buffer to read from, and a single byte
-as EOF (defaults to an empty string)."
-  (setv buff "")
-  (while True
-    (setv inn (string (.readline from-file)))
-    (if (= inn eof)
-      (raise (EOFError "Reached end of file")))
-    (+= buff inn)
-    (try
-      (setv parsed (first (tokenize buff)))
-      (except [e [PrematureEndOfInput IndexError]])
-      (else (break))))
-  parsed)
-
-(defn read-str [input]
-  "Reads and tokenizes first line of `input`."
-  (read :from-file (StringIO input)))
-
 (defn keyword [value]
   "Create a keyword from `value`.
 
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 2f9154e..0e6ab5f 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -8,6 +8,11 @@
 
 (import [hy.models [HyList HySymbol]])
 
+(eval-and-compile
+  (import [hy.core.language [*]]))
+
+(require [hy.core.bootstrap [*]])
+
 (defmacro as-> [head name &rest rest]
   "Beginning with `head`, expand a sequence of assignments `rest` to `name`.
 
diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy
index ecf42a5..9fae8fe 100644
--- a/hy/core/shadow.hy
+++ b/hy/core/shadow.hy
@@ -7,6 +7,8 @@
 (import operator)
 (import [hy._compat [PY3 PY35]])
 
+(require [hy.core.bootstrap [*]])
+
 (if PY3
   (import [functools [reduce]]))
 
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index 093fa49..2103275 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -5,12 +5,18 @@
 from __future__ import unicode_literals
 
 import re
+import sys
 import unicodedata
 
 from hy._compat import str_type, isidentifier, UCS4
-from hy.lex.exceptions import LexException  # NOQA
+from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
 
+try:
+    from io import StringIO
+except ImportError:
+    from StringIO import StringIO
+
 
 def hy_parse(source):
     """Parse a Hy source string.
@@ -122,3 +128,28 @@ def unicode_to_ucs4iter(ustr):
             ucs4_list[i] += ucs4_list[i + 1]
             del ucs4_list[i + 1]
     return ucs4_list
+
+
+def read(from_file=sys.stdin, eof=""):
+    """Read from input and returns a tokenized string.
+
+    Can take a given input buffer to read from, and a single byte as EOF
+    (defaults to an empty string).
+    """
+    buff = ""
+    while True:
+        inn = str(from_file.readline())
+        if inn == eof:
+            raise EOFError("Reached end of file")
+        buff += inn
+        try:
+            parsed = next(iter(tokenize(buff)), None)
+        except (PrematureEndOfInput, IndexError):
+            pass
+        else:
+            break
+    return parsed
+
+
+def read_str(input):
+    return read(StringIO(str_type(input)))

From 15c68455ec7b939da435bb90e655af6167d07177 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sat, 10 Nov 2018 22:43:39 -0600
Subject: [PATCH 085/223] Use a fixed compiler in `HyREPL`

These changes make `HyREPL` use a single `HyASTCompiler` instance, instead of
creating one every time a valid source string is processed.

This change avoids the unnecessary re-initiation of the standard library
`import` and `require` steps that currently occur within the module tracked by a
`HyREPL` instance.

Also, one can now pass an existing compiler instance to `hy_repl` and
`hy_compiler`.

Closes hylang/hy#1698.
---
 hy/cmdline.py  | 10 +++++--
 hy/compiler.py | 72 +++++++++++++++++++++++++++++++-------------------
 2 files changed, 53 insertions(+), 29 deletions(-)

diff --git a/hy/cmdline.py b/hy/cmdline.py
index 7fdfb49..4c98ef1 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -20,7 +20,8 @@ import astor.code_gen
 import hy
 from hy.lex import hy_parse, mangle
 from hy.lex.exceptions import LexException, PrematureEndOfInput
-from hy.compiler import HyTypeError, hy_compile, hy_eval
+from hy.compiler import HyASTCompiler, hy_compile, hy_eval
+from hy.errors import HyTypeError
 from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
@@ -69,6 +70,8 @@ class HyREPL(code.InteractiveConsole, object):
         # Load cmdline-specific macros.
         require('hy.cmdline', module_name, assignments='ALL')
 
+        self.hy_compiler = HyASTCompiler(self.module)
+
         self.spy = spy
 
         if output_fn is None:
@@ -117,7 +120,10 @@ class HyREPL(code.InteractiveConsole, object):
                     new_ast = ast.Module(main_ast.body +
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
-            value = hy_eval(do, self.locals, self.module, ast_callback)
+
+            value = hy_eval(do, self.locals,
+                            ast_callback=ast_callback,
+                            compiler=self.hy_compiler)
         except HyTypeError as e:
             if e.source is None:
                 e.source = source
diff --git a/hy/compiler.py b/hy/compiler.py
index 11473d7..1b5e983 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1779,18 +1779,38 @@ class HyASTCompiler(object):
         return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2])
 
 
-def hy_eval(hytree, locals=None, module=None, ast_callback=None):
-    """Evaluates a quoted expression and returns the value.
+def get_compiler_module(module=None, compiler=None, calling_frame=False):
+    """Get a module object from a compiler, given module object,
+    string name of a module, and (optionally) the calling frame; otherwise,
+    raise an error."""
 
+    module = getattr(compiler, 'module', None) or module
+
+    if isinstance(module, string_types):
+        if module.startswith('<') and module.endswith('>'):
+            module = types.ModuleType(module)
+        else:
+            module = importlib.import_module(ast_str(module, piecewise=True))
+
+    if calling_frame and not module:
+        module = calling_module(n=2)
+
+    if not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    return module
+
+
+def hy_eval(hytree, locals=None, module=None, ast_callback=None,
+            compiler=None):
+    """Evaluates a quoted expression and returns the value.
     Examples
     --------
-
        => (eval '(print "Hello World"))
        "Hello World"
 
     If you want to evaluate a string, use ``read-str`` to convert it to a
     form first:
-
        => (eval (read-str "(+ 1 1)"))
        2
 
@@ -1806,25 +1826,25 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None):
     module: str or types.ModuleType, optional
         Module, or name of the module, to which the Hy tree is assigned and
         the global values are taken.
-        Defaults to the calling frame's module, if any, and '__eval__'
-        otherwise.
+        The module associated with `compiler` takes priority over this value.
+        When neither `module` nor `compiler` is specified, the calling frame's
+        module is used.
 
     ast_callback: callable, optional
         A callback that is passed the Hy compiled tree and resulting
         expression object, in that order, after compilation but before
         evaluation.
 
+    compiler: HyASTCompiler, optional
+        An existing Hy compiler to use for compilation.  Also serves as
+        the `module` value when given.
+
     Returns
     -------
     out : Result of evaluating the Hy compiled tree.
     """
-    if module is None:
-        module = calling_module()
 
-    if isinstance(module, string_types):
-        module = importlib.import_module(ast_str(module, piecewise=True))
-    elif not inspect.ismodule(module):
-        raise TypeError('Invalid module type: {}'.format(type(module)))
+    module = get_compiler_module(module, compiler, True)
 
     if locals is None:
         frame = inspect.stack()[1][0]
@@ -1833,7 +1853,8 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None):
     if not isinstance(locals, dict):
         raise TypeError("Locals must be a dictionary")
 
-    _ast, expr = hy_compile(hytree, module, get_expr=True)
+    _ast, expr = hy_compile(hytree, module=module, get_expr=True,
+                            compiler=compiler)
 
     # Spoof the positions in the generated ast...
     for node in ast.walk(_ast):
@@ -1856,14 +1877,15 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None):
     return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
 
 
-def hy_compile(tree, module, root=ast.Module, get_expr=False):
-    """
-    Compile a Hy tree into a Python AST tree.
+def hy_compile(tree, module=None, root=ast.Module, get_expr=False,
+               compiler=None):
+    """Compile a Hy tree into a Python AST tree.
 
     Parameters
     ----------
-    module: str or types.ModuleType
+    module: str or types.ModuleType, optional
         Module, or name of the module, in which the Hy tree is evaluated.
+        The module associated with `compiler` takes priority over this value.
 
     root: ast object, optional (ast.Module)
         Root object for the Python AST tree.
@@ -1871,26 +1893,22 @@ def hy_compile(tree, module, root=ast.Module, get_expr=False):
     get_expr: bool, optional (False)
         If true, return a tuple with `(root_obj, last_expression)`.
 
+    compiler: HyASTCompiler, optional
+        An existing Hy compiler to use for compilation.  Also serves as
+        the `module` value when given.
+
     Returns
     -------
     out : A Python AST tree
     """
-
-    if isinstance(module, string_types):
-        if module.startswith('<') and module.endswith('>'):
-            module = types.ModuleType(module)
-        else:
-            module = importlib.import_module(ast_str(module, piecewise=True))
-    if not inspect.ismodule(module):
-        raise TypeError('Invalid module type: {}'.format(type(module)))
-
+    module = get_compiler_module(module, compiler, False)
 
     tree = wrap_value(tree)
     if not isinstance(tree, HyObject):
         raise HyCompileError("`tree` must be a HyObject or capable of "
                              "being promoted to one")
 
-    compiler = HyASTCompiler(module)
+    compiler = compiler or HyASTCompiler(module)
     result = compiler.compile(tree)
     expr = result.force_expr
 

From 3d0659b7231dcdeef86fbecd1584ec3aa0e1c432 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sun, 11 Nov 2018 15:27:43 -0600
Subject: [PATCH 086/223] Update NEWS

---
 NEWS.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 44ab3cb..3355da7 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -9,6 +9,8 @@ Removals
 
 New Features
 ------------------------------
+* `eval` / `hy_eval` and `hy_compile` now accept an optional `compiler` argument
+  that enables the use of an existing `HyASTCompiler` instance.
 * Keyword objects (not just literal keywords) can be called, as
   shorthand for `(get obj :key)`, and they accept a default value
   as a second argument.

From 690416b3d6ab7c7194050f772239b53c4b2d6f86 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Wed, 28 Nov 2018 16:56:00 -0600
Subject: [PATCH 087/223] Update description of `eval` in core.rst

---
 docs/language/core.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/language/core.rst b/docs/language/core.rst
index d292d75..4237418 100644
--- a/docs/language/core.rst
+++ b/docs/language/core.rst
@@ -195,7 +195,9 @@ eval
 ``eval`` evaluates a quoted expression and returns the value. The optional
 second and third arguments specify the dictionary of globals to use and the
 module name. The globals dictionary defaults to ``(local)`` and the module name
-defaults to the name of the current module.
+defaults to the name of the current module.  An optional fourth keyword parameter,
+``compiler``, allows one to re-use an existing ``HyASTCompiler`` object for the
+compilation step.
 
 .. code-block:: clj
 
@@ -1403,4 +1405,3 @@ are available. Some of their names have been changed:
   - ``dropwhile`` has been changed to ``drop-while``
 
   - ``filterfalse`` has been changed to ``remove``
-

From b8b02c9df9e58808d9c7f6d1c048fc87f9cb65db Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Mon, 26 Nov 2018 13:17:32 -0600
Subject: [PATCH 088/223] Handle empty `defmain` args

Closes hylang/hy#1707.
---
 hy/core/macros.hy                   | 12 ++++++++---
 tests/native_tests/native_macros.hy | 32 +++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 0e6ab5f..f8ce33b 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -223,11 +223,17 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
 
 
 (defmacro defmain [args &rest body]
-  "Write a function named \"main\" and do the 'if __main__' dance"
-  (setv retval (gensym))
+  "Write a function named \"main\" and do the 'if __main__' dance.
+
+The symbols in `args` are bound to the elements in `sys.argv`, which means that
+the first symbol in `args` will always take the value of the script/executable
+name (i.e. `sys.argv[0]`).
+"
+  (setv retval (gensym)
+        restval (gensym))
   `(when (= --name-- "__main__")
      (import sys)
-     (setv ~retval ((fn [~@args] ~@body) #* sys.argv))
+     (setv ~retval ((fn [~@(or args `[&rest ~restval])] ~@body) #* sys.argv))
      (if (integer? ~retval)
        (sys.exit ~retval))))
 
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index fc17edb..b5cb979 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -325,14 +325,38 @@
   (global --name--)
   (setv oldname --name--)
   (setv --name-- "__main__")
-  (defn main []
-    (print 'Hy)
-    42)
+
+  (defn main [x]
+    (print (integer? x))
+    x)
+
   (try
     (defmain [&rest args]
-      (main))
+      (main 42))
+    (assert False)
     (except [e SystemExit]
       (assert (= (str e) "42"))))
+
+  ;; Try a `defmain` without args
+  (try
+    (defmain []
+      (main 42))
+    (assert False)
+    (except [e SystemExit]
+      (assert (= (str e) "42"))))
+
+  ;; Try a `defmain` with only one arg
+  (import sys)
+  (setv oldargv sys.argv)
+  (try
+    (setv sys.argv [1])
+    (defmain [x]
+      (main x))
+    (assert False)
+    (except [e SystemExit]
+      (assert (= (str e) "1"))))
+
+  (setv sys.argv oldargv)
   (setv --name-- oldname))
 
 (defn test-macro-namespace-resolution []

From f040bbe96f2bf7d1ef578f03ba6278c35a325ddc Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Tue, 27 Nov 2018 14:27:42 -0600
Subject: [PATCH 089/223] Update NEWS

---
 NEWS.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS.rst b/NEWS.rst
index 3355da7..036843a 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -21,6 +21,7 @@ New Features
 
 Bug Fixes
 ------------------------------
+* Fixed issue with empty arguments in `defmain`.
 * `require` now compiles to Python AST.
 * Fixed circular `require`s.
 * Fixed module reloading.

From b777972a0e9eb9f5077cc739e6f0bfcdf83311a7 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 14 Dec 2018 15:54:23 -0500
Subject: [PATCH 090/223] Fix mangling of characters below 0xFF

---
 NEWS.rst                       |  1 +
 hy/lex/__init__.py             |  5 ++++-
 tests/native_tests/mangling.hy | 21 +++++++++++++++++++++
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/NEWS.rst b/NEWS.rst
index 036843a..518ca7a 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -34,6 +34,7 @@ Bug Fixes
 * Fixed errors from `from __future__ import ...` statements and missing Hy
   module docstrings caused by automatic importing of Hy builtins.
 * Fixed crash in `mangle` for some pathological inputs
+* Fixed incorrect mangling of some characters at low code points
 
 0.15.0
 ==============================
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index 2103275..d1bfb5e 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -61,7 +61,10 @@ def mangle(s):
     according to Hy's mangling rules."""
     def unicode_char_to_hex(uchr):
         # Covert a unicode char to hex string, without prefix
-        return uchr.encode('unicode-escape').decode('utf-8').lstrip('\\U').lstrip('\\u').lstrip('0')
+        if len(uchr) == 1 and ord(uchr) < 128:
+            return format(ord(uchr), 'x')
+        return (uchr.encode('unicode-escape').decode('utf-8')
+            .lstrip('\\U').lstrip('\\u').lstrip('\\x').lstrip('0'))
 
     assert s
 
diff --git a/tests/native_tests/mangling.hy b/tests/native_tests/mangling.hy
index bfb4d0a..d3f6eae 100644
--- a/tests/native_tests/mangling.hy
+++ b/tests/native_tests/mangling.hy
@@ -158,6 +158,27 @@
     (assert (= (mangle a) b))
     (assert (= (unmangle b) a))))
 
+
+(defn test-nongraphic []
+  ; https://github.com/hylang/hy/issues/1694
+
+  (assert (= (mangle " ") "hyx_XspaceX"))
+  (assert (= (mangle "\a") "hyx_XU7X"))
+  (assert (= (mangle "\t") "hyx_XU9X"))
+  (assert (= (mangle "\n") "hyx_XUaX"))
+  (assert (= (mangle "\r") "hyx_XUdX"))
+  (assert (= (mangle "\r") "hyx_XUdX"))
+
+  (setv c (try unichr (except [NameError] chr)))
+  (assert (= (mangle (c 127)) "hyx_XU7fX"))
+  (assert (= (mangle (c 128)) "hyx_XU80X"))
+  (assert (= (mangle (c 0xa0)) "hyx_XnoHbreak_spaceX"))
+  (assert (= (mangle (c 0x378)) "hyx_XU378X"))
+  (assert (= (mangle (c 0x200a) "hyx_Xhair_spaceX")))
+  (assert (= (mangle (c 0x2065)) "hyx_XU2065X"))
+  (assert (= (mangle (c 0x1000c)) "hyx_XU1000cX")))
+
+
 (defn test-mangle-bad-indent []
   ; Shouldn't crash with IndentationError
   (mangle "  0\n 0"))

From 70747a58c3f809d41b3a93bcf21df113241f2701 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 26 Dec 2018 10:00:56 -0500
Subject: [PATCH 091/223] Fix an example in interop.rst

---
 docs/language/interop.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/language/interop.rst b/docs/language/interop.rst
index 34d61ea..6276659 100644
--- a/docs/language/interop.rst
+++ b/docs/language/interop.rst
@@ -47,8 +47,8 @@ If you save the following in ``greetings.hy``:
 
 .. code-block:: clj
 
-    (setv *this-will-be-in-caps-and-underscores* "See?")
-    (defn greet [name] (print "hello from hy," name))
+    (setv this-will-have-underscores "See?")
+    (defn greet [name] (print "Hello from Hy," name))
 
 Then you can use it directly from Python, by importing Hy before importing
 the module. In Python::
@@ -56,8 +56,8 @@ the module. In Python::
     import hy
     import greetings
 
-    greetings.greet("Foo") # prints "Hello from hy, Foo"
-    print(THIS_WILL_BE_IN_CAPS_AND_UNDERSCORES) # prints "See?"
+    greetings.greet("Foo") # prints "Hello from Hy, Foo"
+    print(greetings.this_will_have_underscores) # prints "See?"
 
 If you create a package with Hy code, and you do the ``import hy`` in
 ``__init__.py``, you can then directly include the package. Of course, Hy still

From 3d2be62d4b3fa6c43a6f11f38c10b2d7e736f6eb Mon Sep 17 00:00:00 2001
From: Eli <elimaynard923@gmail.com>
Date: Thu, 17 Jan 2019 08:05:23 -0500
Subject: [PATCH 092/223] Add synonyms for argument unpacking, for text-search
 purposes

---
 docs/language/api.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index ab0c041..115460e 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -1609,6 +1609,9 @@ the given conditional is ``False``. The following shows the expansion of this ma
 unpack-iterable, unpack-mapping
 -------------------------------
 
+(Also known as the splat operator, star operator, argument expansion, argument
+explosion, argument gathering, and varargs, among others...)
+
 ``unpack-iterable`` and ``unpack-mapping`` allow an iterable or mapping
 object (respectively) to provide positional or keywords arguments
 (respectively) to a function.

From e4fd74af1b94dd22e6f8e4f8b3752f17c9391fc6 Mon Sep 17 00:00:00 2001
From: Eli <elimaynard923@gmail.com>
Date: Thu, 17 Jan 2019 08:16:32 -0500
Subject: [PATCH 093/223] Clarifying &optional documentation (fixes #1722)

---
 docs/language/api.rst | 84 +++++++++++++++++++++++++++++++++----------
 1 file changed, 65 insertions(+), 19 deletions(-)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index 115460e..4f52418 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -467,25 +467,35 @@ and `&kwonly` must precede `&kwargs`). This is the same order that Python
 requires.
 
 &optional
-    Parameter is optional. The parameter can be given as a two item list, where
-    the first element is parameter name and the second is the default value. The
-    parameter can be also given as a single item, in which case the default
-    value is ``None``.
+    All following parameters are optional. They may be given as two-argument lists,
+    where the first element is the parameter name and the second is the default value.
+    The parameter can also be given as a single item, in which case the default value
+    is ``None``.
+
+    The following example defines a function with one required positional argument
+    as well as three optional arguments. The first optional argument defaults to ``None``
+    and the latter two default to ``"("`` and ``")"``, respectively.
 
     .. code-block:: clj
 
-        => (defn total-value [value &optional [value-added-tax 10]]
-        ...  (+ (/ (* value value-added-tax) 100) value))
+      => (defn format-pair [left-val &optional right-val  [open-text "("] [close-text ")"]]
+      ...  (+ open-text (str left-val) ", " (str right-val) close-text))
 
-        => (total-value 100)
-        110.0
+      => (format-pair 3)
+      '(3, None)'
 
-        => (total-value 100 1)
-        101.0
+      => (format-pair "A" "B")
+      '(A, B)'
+
+      => (format-pair "A" "B" "<" ">")
+      '<A, B>'
+
+      => (format-pair "A" :open-text "<" :close-text ">")
+      '<A, None>'
 
 &rest
-    Parameter will contain 0 or more positional arguments. No other positional
-    arguments may be specified after this one.
+    The following parameter will contain a list of 0 or more positional arguments.
+    No other positional parameters may be specified after this one.
 
     The following code example defines a function that can be given 0 to n
     numerical parameters. It then sums every odd number and subtracts
@@ -504,15 +514,14 @@ requires.
         8
         => (zig-zag-sum 1 2 3 4 5 6)
         -3
-		
+
 &kwonly
     .. versionadded:: 0.12.0
 
-    Parameters that can only be called as keywords. Mandatory
-    keyword-only arguments are declared with the argument's name;
-    optional keyword-only arguments are declared as a two-element list
-    containing the argument name followed by the default value (as
-    with `&optional` above).
+    All following parmaeters can only be supplied as keywords.
+    Like ``&optional``, the parameter may be marked as optional by
+    declaring it as a two-element list containing the parameter name
+    following by the default value.
 
     .. code-block:: clj
 
@@ -541,7 +550,8 @@ requires.
     Availability: Python 3.
 
 &kwargs
-    Parameter will contain 0 or more keyword arguments.
+    Like ``&rest``, but for keyword arugments.
+    The following parameter will contain 0 or more keyword arguments.
 
     The following code examples defines a function that will print all keyword
     arguments and their values.
@@ -560,6 +570,42 @@ requires.
         parameter-1 1
         parameter-2 2
 
+The following example uses all of ``&optional``, ``&rest``, ``&kwonly``, and
+``&kwargs`` in order to show their interactions with each other. The function
+renders an HTML tag.
+It requires an argument ``tag-name``, a string which is the tag name.
+It has one optional argument, ``delim``, which defaults to ``""`` and is placed
+between each child.
+The rest of the arguments, ``children``, are the tag's children or content.
+A single keyword-only argument, ``empty``, is included and defaults to ``False``.
+``empty`` changes how the tag is rendered if it has no children. Normally, a
+tag with no children is rendered like ``<div></div>``. If ``empty`` is ``True``,
+then it will render like ``<div />``.
+The rest of the keyword arguments, ``props``, render as HTML attributes.
+
+.. code-block:: clj
+
+  => (defn render-html-tag [tag-name &optional [delim ""] &rest children &kwonly [empty False] &kwargs attrs]
+  ...  (setv rendered-attrs (.join " " (lfor (, key val) (.items attrs) (+ (unmangle (str key)) "=\"" (str val) "\""))))
+  ...  (if rendered-attrs  ; If we have attributes, prefix them with a space after the tag name
+  ...    (setv rendered-attrs (+ " " rendered-attrs)))
+  ...  (setv rendered-children (.join delim children))
+  ...  (if (and (not children) empty)
+  ...    (+ "<" tag-name rendered-attrs " />")
+  ...    (+ "<" tag-name rendered-attrs ">" rendered-children "</" tag-name ">")))
+
+  => (render-html-tag "div")
+  '<div></div'>
+
+  => (render-html-tag "img" :empty True)
+  '<img />'
+
+  => (render-html-tag "img" :id "china" :class "big-image" :empty True)
+  '<img id="china" class="big-image" />'
+
+  => (render-html-tag "p" " --- " (render-html-tag "span" "" :class "fancy" "I'm fancy!") "I'm to the right of fancy" "I'm alone :(")
+  '<p><span class="fancy">I\'m fancy!</span> --- I\'m to right right of fancy --- I\'m alone :(</p>'
+
 defn/a
 ------
 

From 983ea2dda24cc0a950410009818122a91158b59e Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 16 Jan 2019 08:02:01 -0500
Subject: [PATCH 094/223] Make (require [foo [*]]) pull in macros required by
 `foo`

---
 hy/macros.py                        |  9 ++-------
 tests/native_tests/native_macros.hy | 12 ++++++++----
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/hy/macros.py b/hy/macros.py
index 0a6bc1b..22b0ce3 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -190,14 +190,9 @@ def require(source_module, target_module, assignments, prefix=""):
         prefix += "."
 
     if assignments == "ALL":
-        # Only add macros/tags created in/by the source module.
-        name_assigns = [(n, n) for n, f in source_macros.items()
-                        if inspect.getmodule(f) == source_module]
-        name_assigns += [(n, n) for n, f in source_tags.items()
-                         if inspect.getmodule(f) == source_module]
+        name_assigns = [(k, k) for k in
+            tuple(source_macros.keys()) + tuple(source_tags.keys())]
     else:
-        # If one specifically requests a macro/tag not created in the source
-        # module, I guess we allow it?
         name_assigns = assignments
 
     for name, alias in name_assigns:
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index b5cb979..75c8816 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -378,10 +378,6 @@ in expansions."
 
   (require [tests.resources.macro-with-require [*]])
 
-  ;; Make sure our local version wasn't overwritten by a faulty `require` of the
-  ;; one in tests.resources.macro-with-require.
-  (assert (= nonlocal-test-macro (get __macros__ "nonlocal_test_macro")))
-
   (setv module-name-var "tests.native_tests.native_macros.test-macro-namespace-resolution")
   (assert (= (+ "This macro was created in tests.resources.macros, "
                 "expanded in tests.native_tests.native_macros.test-macro-namespace-resolution "
@@ -479,3 +475,11 @@ in expansions."
   ;; ensure that an imported module used the cached bytecode.  We'll simply have
   ;; to trust that the .pyc loading convention was followed.
   (test-requires-and-macros))
+
+
+(defn test-recursive-require-star []
+  "(require [foo [*]]) should pull in macros required by `foo`."
+  (require [tests.resources.macro-with-require [*]])
+
+  (test-macro)
+  (assert (= blah 1)))

From f1e693c96b4dbd03f5b702c55f08a04685abbf2a Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 20 Jan 2019 11:59:49 -0500
Subject: [PATCH 095/223] Fix a Python 2 crash

---
 NEWS.rst      | 2 ++
 hy/_compat.py | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 518ca7a..6f3d53c 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -35,6 +35,8 @@ Bug Fixes
   module docstrings caused by automatic importing of Hy builtins.
 * Fixed crash in `mangle` for some pathological inputs
 * Fixed incorrect mangling of some characters at low code points
+* Fixed a crash on certain versions of Python 2 due to changes
+  in the standard module `tokenize`
 
 0.15.0
 ==============================
diff --git a/hy/_compat.py b/hy/_compat.py
index 379ce43..ddd6c48 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -47,6 +47,11 @@ def isidentifier(x):
         tokens = list(T.generate_tokens(StringIO(x).readline))
     except (T.TokenError, IndentationError):
         return False
+    # Some versions of Python 2.7 (including one that made it into
+    # Ubuntu 18.10) have a Python 3 backport that adds a NEWLINE
+    # token. Remove it if it's present.
+    # https://bugs.python.org/issue33899
+    tokens = [t for t in tokens if t[0] != T.NEWLINE]
     return len(tokens) == 2 and tokens[0][0] == T.NAME
 
 try:

From d7c6fd8fa176f52c3d20f8ccde2fa608e878e7c0 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 21 Jan 2019 19:48:54 -0500
Subject: [PATCH 096/223] Upgrade rply to avoid a deprecation warning

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a8a28ca..4844255 100755
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,7 @@ class Install(install):
                         "." + filename[:-len(".hy")])
         install.run(self)
 
-install_requires = ['rply>=0.7.6', 'astor>=0.7.1', 'funcparserlib>=0.3.6', 'clint>=0.4']
+install_requires = ['rply>=0.7.7', 'astor>=0.7.1', 'funcparserlib>=0.3.6', 'clint>=0.4']
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 

From 7d72e2fe74ff0cd815c02a0ad93eb7aa08f7f9ff Mon Sep 17 00:00:00 2001
From: edouardklein <edouardk.lei.n@gmail.com>
Date: Thu, 24 Jan 2019 17:57:10 +0100
Subject: [PATCH 097/223] Correct documentation of earmuff behavior

Another piece of fix #1714
---
 docs/style-guide.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/style-guide.rst b/docs/style-guide.rst
index 2d40326..68f2eae 100644
--- a/docs/style-guide.rst
+++ b/docs/style-guide.rst
@@ -36,8 +36,7 @@ The Tao of Hy
 The following illustrates a brief list of design decisions that went
 into the making of Hy.
 
-+ Look like a Lisp; DTRT with it (e.g. dashes turn to underscores, earmuffs
-  turn to all-caps).
++ Look like a Lisp; DTRT with it (e.g. dashes turn to underscores).
 + We're still Python. Most of the internals translate 1:1 to Python internals.
 + Use Unicode everywhere.
 + Fix the bad decisions in Python 2 when we can (see ``true_division``).

From 62638b44a3123e5c2edee808f4574bf61b7b5e5f Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 7 Feb 2019 08:57:35 -0500
Subject: [PATCH 098/223] Update copyright years

---
 LICENSE                                 | 2 +-
 hy/_compat.py                           | 2 +-
 hy/cmdline.py                           | 2 +-
 hy/compiler.py                          | 2 +-
 hy/completer.py                         | 2 +-
 hy/contrib/botsbuildbots.hy             | 2 +-
 hy/contrib/hy_repr.hy                   | 2 +-
 hy/contrib/loop.hy                      | 2 +-
 hy/contrib/multi.hy                     | 2 +-
 hy/contrib/profile.hy                   | 2 +-
 hy/contrib/sequences.hy                 | 2 +-
 hy/contrib/walk.hy                      | 2 +-
 hy/core/bootstrap.hy                    | 2 +-
 hy/core/language.hy                     | 2 +-
 hy/core/macros.hy                       | 2 +-
 hy/core/shadow.hy                       | 2 +-
 hy/errors.py                            | 2 +-
 hy/extra/anaphoric.hy                   | 2 +-
 hy/extra/reserved.hy                    | 2 +-
 hy/importer.py                          | 2 +-
 hy/lex/__init__.py                      | 2 +-
 hy/lex/exceptions.py                    | 2 +-
 hy/lex/lexer.py                         | 2 +-
 hy/lex/parser.py                        | 2 +-
 hy/macros.py                            | 2 +-
 hy/model_patterns.py                    | 2 +-
 hy/models.py                            | 2 +-
 scripts/update-coreteam.hy              | 2 +-
 setup.py                                | 2 +-
 tests/compilers/test_ast.py             | 2 +-
 tests/compilers/test_compiler.py        | 2 +-
 tests/importer/test_importer.py         | 2 +-
 tests/importer/test_pyc.py              | 2 +-
 tests/macros/test_macro_processor.py    | 2 +-
 tests/macros/test_tag_macros.py         | 2 +-
 tests/native_tests/contrib/hy_repr.hy   | 2 +-
 tests/native_tests/contrib/loop.hy      | 2 +-
 tests/native_tests/contrib/multi.hy     | 2 +-
 tests/native_tests/contrib/sequences.hy | 2 +-
 tests/native_tests/contrib/walk.hy      | 2 +-
 tests/native_tests/core.hy              | 2 +-
 tests/native_tests/defclass.hy          | 2 +-
 tests/native_tests/extra/anaphoric.hy   | 2 +-
 tests/native_tests/extra/reserved.hy    | 2 +-
 tests/native_tests/language.hy          | 2 +-
 tests/native_tests/mangling.hy          | 2 +-
 tests/native_tests/mathematics.hy       | 2 +-
 tests/native_tests/model_patterns.hy    | 2 +-
 tests/native_tests/native_macros.hy     | 2 +-
 tests/native_tests/operators.hy         | 2 +-
 tests/native_tests/py35_only_tests.hy   | 2 +-
 tests/native_tests/py36_only_tests.hy   | 2 +-
 tests/native_tests/py3_only_tests.hy    | 2 +-
 tests/native_tests/quote.hy             | 2 +-
 tests/native_tests/tag_macros.hy        | 2 +-
 tests/native_tests/with_decorator.hy    | 2 +-
 tests/native_tests/with_test.hy         | 2 +-
 tests/resources/argparse_ex.hy          | 2 +-
 tests/resources/pydemo.hy               | 2 +-
 tests/test_bin.py                       | 2 +-
 tests/test_hy2py.py                     | 2 +-
 tests/test_lex.py                       | 2 +-
 tests/test_models.py                    | 2 +-
 63 files changed, 63 insertions(+), 63 deletions(-)

diff --git a/LICENSE b/LICENSE
index dd23317..23797bc 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright 2018 the authors.
+Copyright 2019 the authors.
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the "Software"),
diff --git a/hy/_compat.py b/hy/_compat.py
index ddd6c48..bd9390f 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/cmdline.py b/hy/cmdline.py
index 4c98ef1..4c2e1c3 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/compiler.py b/hy/compiler.py
index 1b5e983..f3b55a3 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/completer.py b/hy/completer.py
index 9b7bb4f..38cf8eb 100644
--- a/hy/completer.py
+++ b/hy/completer.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/contrib/botsbuildbots.hy b/hy/contrib/botsbuildbots.hy
index 07dee93..5088cca 100644
--- a/hy/contrib/botsbuildbots.hy
+++ b/hy/contrib/botsbuildbots.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/contrib/hy_repr.hy b/hy/contrib/hy_repr.hy
index 5d238ba..9fd59be 100644
--- a/hy/contrib/hy_repr.hy
+++ b/hy/contrib/hy_repr.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/contrib/loop.hy b/hy/contrib/loop.hy
index 6e1f29f..b1cb2da 100644
--- a/hy/contrib/loop.hy
+++ b/hy/contrib/loop.hy
@@ -1,5 +1,5 @@
 ;;; Hy tail-call optimization
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/contrib/multi.hy b/hy/contrib/multi.hy
index 7867804..7dba654 100644
--- a/hy/contrib/multi.hy
+++ b/hy/contrib/multi.hy
@@ -1,5 +1,5 @@
 ;; Hy Arity-overloading
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/contrib/profile.hy b/hy/contrib/profile.hy
index 06fec24..8ff2ed6 100644
--- a/hy/contrib/profile.hy
+++ b/hy/contrib/profile.hy
@@ -1,5 +1,5 @@
 ;;; Hy profiling macros
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/contrib/sequences.hy b/hy/contrib/sequences.hy
index e684b80..94b3236 100644
--- a/hy/contrib/sequences.hy
+++ b/hy/contrib/sequences.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index 70fbca5..a7a46e9 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -1,5 +1,5 @@
 ;;; Hy AST walker
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy
index cc576bc..fa613a6 100644
--- a/hy/core/bootstrap.hy
+++ b/hy/core/bootstrap.hy
@@ -1,5 +1,5 @@
 ;;; Hy bootstrap macros
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/core/language.hy b/hy/core/language.hy
index e13cbfa..ac046bb 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index f8ce33b..824a8e5 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -1,5 +1,5 @@
 ;;; Hy core macros
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy
index 9fae8fe..9560d97 100644
--- a/hy/core/shadow.hy
+++ b/hy/core/shadow.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/errors.py b/hy/errors.py
index a60b09c..7d36ab2 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/extra/anaphoric.hy b/hy/extra/anaphoric.hy
index 276f72d..602d9ef 100644
--- a/hy/extra/anaphoric.hy
+++ b/hy/extra/anaphoric.hy
@@ -1,5 +1,5 @@
 ;;; Hy anaphoric macros
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/extra/reserved.hy b/hy/extra/reserved.hy
index 3de34ea..81d8a81 100644
--- a/hy/extra/reserved.hy
+++ b/hy/extra/reserved.hy
@@ -1,5 +1,5 @@
 ;;; Get a frozenset of Hy reserved words
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/hy/importer.py b/hy/importer.py
index 5e68c61..e3cc50d 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index d1bfb5e..bf82cf9 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/lex/exceptions.py b/hy/lex/exceptions.py
index 14df78b..573a8e8 100644
--- a/hy/lex/exceptions.py
+++ b/hy/lex/exceptions.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/lex/lexer.py b/hy/lex/lexer.py
index a3fecac..14b7c88 100755
--- a/hy/lex/lexer.py
+++ b/hy/lex/lexer.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index a13c793..c602734 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/macros.py b/hy/macros.py
index 22b0ce3..be59200 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 import importlib
diff --git a/hy/model_patterns.py b/hy/model_patterns.py
index 5291768..45124ba 100644
--- a/hy/model_patterns.py
+++ b/hy/model_patterns.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/hy/models.py b/hy/models.py
index 2ff5efa..cf02dab 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/scripts/update-coreteam.hy b/scripts/update-coreteam.hy
index cc59630..6e1036f 100644
--- a/scripts/update-coreteam.hy
+++ b/scripts/update-coreteam.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/setup.py b/setup.py
index 4844255..54aba2b 100755
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 1f94c3f..75e9c49 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/compilers/test_compiler.py b/tests/compilers/test_compiler.py
index cbce371..bbb2970 100644
--- a/tests/compilers/test_compiler.py
+++ b/tests/compilers/test_compiler.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index 1973fe8..3017d16 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/importer/test_pyc.py b/tests/importer/test_pyc.py
index ad37617..287e9e3 100644
--- a/tests/importer/test_pyc.py
+++ b/tests/importer/test_pyc.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py
index 8644532..309ca49 100644
--- a/tests/macros/test_macro_processor.py
+++ b/tests/macros/test_macro_processor.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/macros/test_tag_macros.py b/tests/macros/test_tag_macros.py
index f9c4f69..51f57e8 100644
--- a/tests/macros/test_tag_macros.py
+++ b/tests/macros/test_tag_macros.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/native_tests/contrib/hy_repr.hy b/tests/native_tests/contrib/hy_repr.hy
index 2944d85..7867181 100644
--- a/tests/native_tests/contrib/hy_repr.hy
+++ b/tests/native_tests/contrib/hy_repr.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/contrib/loop.hy b/tests/native_tests/contrib/loop.hy
index 1c145b9..f5c425c 100644
--- a/tests/native_tests/contrib/loop.hy
+++ b/tests/native_tests/contrib/loop.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/contrib/multi.hy b/tests/native_tests/contrib/multi.hy
index 0bc3f58..ab4d85c 100644
--- a/tests/native_tests/contrib/multi.hy
+++ b/tests/native_tests/contrib/multi.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/contrib/sequences.hy b/tests/native_tests/contrib/sequences.hy
index 423d47d..808cf02 100644
--- a/tests/native_tests/contrib/sequences.hy
+++ b/tests/native_tests/contrib/sequences.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index a93e3bf..2e59cfe 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index 496d9bb..a89eece 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/defclass.hy b/tests/native_tests/defclass.hy
index b07e991..2563072 100644
--- a/tests/native_tests/defclass.hy
+++ b/tests/native_tests/defclass.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/extra/anaphoric.hy b/tests/native_tests/extra/anaphoric.hy
index f548704..e7b5fba 100644
--- a/tests/native_tests/extra/anaphoric.hy
+++ b/tests/native_tests/extra/anaphoric.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/extra/reserved.hy b/tests/native_tests/extra/reserved.hy
index 402549d..ca67577 100644
--- a/tests/native_tests/extra/reserved.hy
+++ b/tests/native_tests/extra/reserved.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 3d1c396..c237a5a 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/mangling.hy b/tests/native_tests/mangling.hy
index d3f6eae..832d236 100644
--- a/tests/native_tests/mangling.hy
+++ b/tests/native_tests/mangling.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/mathematics.hy b/tests/native_tests/mathematics.hy
index d03f956..a4a61b0 100644
--- a/tests/native_tests/mathematics.hy
+++ b/tests/native_tests/mathematics.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/model_patterns.hy b/tests/native_tests/model_patterns.hy
index 0e6c316..ce9b835 100644
--- a/tests/native_tests/model_patterns.hy
+++ b/tests/native_tests/model_patterns.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 75c8816..1700b5d 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy
index 990cb33..716eb77 100644
--- a/tests/native_tests/operators.hy
+++ b/tests/native_tests/operators.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/py35_only_tests.hy b/tests/native_tests/py35_only_tests.hy
index 680dcf8..fe17443 100644
--- a/tests/native_tests/py35_only_tests.hy
+++ b/tests/native_tests/py35_only_tests.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy
index d324a26..40de6d9 100644
--- a/tests/native_tests/py36_only_tests.hy
+++ b/tests/native_tests/py36_only_tests.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/py3_only_tests.hy b/tests/native_tests/py3_only_tests.hy
index 1d69546..e9fd24a 100644
--- a/tests/native_tests/py3_only_tests.hy
+++ b/tests/native_tests/py3_only_tests.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/quote.hy b/tests/native_tests/quote.hy
index 3b31350..84371f0 100644
--- a/tests/native_tests/quote.hy
+++ b/tests/native_tests/quote.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/tag_macros.hy b/tests/native_tests/tag_macros.hy
index 4fb3659..34d6592 100644
--- a/tests/native_tests/tag_macros.hy
+++ b/tests/native_tests/tag_macros.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/with_decorator.hy b/tests/native_tests/with_decorator.hy
index 1202eb8..68dd1fc 100644
--- a/tests/native_tests/with_decorator.hy
+++ b/tests/native_tests/with_decorator.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/native_tests/with_test.hy b/tests/native_tests/with_test.hy
index 6f2fc16..8a474a3 100644
--- a/tests/native_tests/with_test.hy
+++ b/tests/native_tests/with_test.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/resources/argparse_ex.hy b/tests/resources/argparse_ex.hy
index e6a5324..70aed9b 100755
--- a/tests/resources/argparse_ex.hy
+++ b/tests/resources/argparse_ex.hy
@@ -1,5 +1,5 @@
 #!/usr/bin/env hy
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
index d8a2447..e98fb6d 100644
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -1,4 +1,4 @@
-;; Copyright 2018 the authors.
+;; Copyright 2019 the authors.
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 6d1e10e..6336122 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- encoding: utf-8 -*-
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
index 6bb0786..6428eef 100644
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/test_lex.py b/tests/test_lex.py
index 2e68876..19da88b 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
diff --git a/tests/test_models.py b/tests/test_models.py
index ddd00de..fd1b615 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -1,4 +1,4 @@
-# Copyright 2018 the authors.
+# Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 

From 902926c543fa57a91b756e95168ded9543660c8c Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sat, 8 Dec 2018 15:36:13 -0500
Subject: [PATCH 099/223] Use ._syntax_error in place of HyTypeError.

And standardize the indentation of these calls.
---
 hy/compiler.py | 96 ++++++++++++++++++++++++--------------------------
 1 file changed, 46 insertions(+), 50 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index f3b55a3..586129e 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -436,6 +436,9 @@ class HyASTCompiler(object):
         except Exception as e:
             raise_empty(HyCompileError, e, sys.exc_info()[2])
 
+    def _syntax_error(self, expr, message):
+        return HyTypeError(expr, message)
+
     def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
                          oldpy_unpack=False):
         """Collect the expression contexts from a list of compiled expression.
@@ -455,8 +458,8 @@ class HyASTCompiler(object):
 
             if not PY35 and oldpy_unpack and is_unpack("iterable", expr):
                 if oldpy_starargs:
-                    raise HyTypeError(expr, "Pythons < 3.5 allow only one "
-                                            "`unpack-iterable` per call")
+                    raise self._syntax_error(expr,
+                        "Pythons < 3.5 allow only one `unpack-iterable` per call")
                 oldpy_starargs = self.compile(expr[1])
                 ret += oldpy_starargs
                 oldpy_starargs = oldpy_starargs.force_expr
@@ -472,21 +475,20 @@ class HyASTCompiler(object):
                             expr, arg=None, value=ret.force_expr))
                 elif oldpy_unpack:
                     if oldpy_kwargs:
-                        raise HyTypeError(expr, "Pythons < 3.5 allow only one "
-                                                "`unpack-mapping` per call")
+                        raise self._syntax_error(expr,
+                            "Pythons < 3.5 allow only one `unpack-mapping` per call")
                     oldpy_kwargs = ret.force_expr
 
             elif with_kwargs and isinstance(expr, HyKeyword):
                 try:
                     value = next(exprs_iter)
                 except StopIteration:
-                    raise HyTypeError(expr,
-                                      "Keyword argument {kw} needs "
-                                      "a value.".format(kw=expr))
+                    raise self._syntax_error(expr,
+                        "Keyword argument {kw} needs a value.".format(kw=expr))
 
                 if not expr:
-                    raise HyTypeError(expr, "Can't call a function with the "
-                                            "empty keyword")
+                    raise self._syntax_error(expr,
+                        "Can't call a function with the empty keyword")
 
                 compiled_value = self.compile(value)
                 ret += compiled_value
@@ -527,8 +529,8 @@ class HyASTCompiler(object):
 
         if isinstance(name, Result):
             if not name.is_expr():
-                raise HyTypeError(expr,
-                                  "Can't assign or delete a non-expression")
+                raise self._syntax_error(expr,
+                    "Can't assign or delete a non-expression")
             name = name.expr
 
         if isinstance(name, (ast.Tuple, ast.List)):
@@ -547,9 +549,8 @@ class HyASTCompiler(object):
             new_name = ast.Starred(
                 value=self._storeize(expr, name.value, func))
         else:
-            raise HyTypeError(expr,
-                              "Can't assign or delete a %s" %
-                              type(expr).__name__)
+            raise self._syntax_error(expr,
+                "Can't assign or delete a %s" % type(expr).__name__)
 
         new_name.ctx = func()
         ast.copy_location(new_name, name)
@@ -575,9 +576,8 @@ class HyASTCompiler(object):
             op = unmangle(ast_str(form[0]))
         if level == 0 and op in ("unquote", "unquote-splice"):
             if len(form) != 2:
-                raise HyTypeError(form,
-                                  ("`%s' needs 1 argument, got %s" %
-                                   op, len(form) - 1))
+                raise HyTypeError("`%s' needs 1 argument, got %s" % op, len(form) - 1,
+                                  self.filename, form, self.source)
             return set(), form[1], op == "unquote-splice"
         elif op == "quasiquote":
             level += 1
@@ -629,7 +629,8 @@ class HyASTCompiler(object):
     @special("unpack-iterable", [FORM])
     def compile_unpack_iterable(self, expr, root, arg):
         if not PY3:
-            raise HyTypeError(expr, "`unpack-iterable` isn't allowed here")
+            raise self._syntax_error(expr,
+                "`unpack-iterable` isn't allowed here")
         ret = self.compile(arg)
         ret += asty.Starred(expr, value=ret.force_expr, ctx=ast.Load())
         return ret
@@ -659,7 +660,8 @@ class HyASTCompiler(object):
 
         if cause is not None:
             if not PY3:
-                raise HyTypeError(expr, "raise from only supported in python 3")
+                raise self._syntax_error(expr,
+                    "raise from only supported in python 3")
             cause = self.compile(cause)
             ret += cause
             cause = cause.force_expr
@@ -706,13 +708,11 @@ class HyASTCompiler(object):
 
         # Using (else) without (except) is verboten!
         if orelse and not handlers:
-            raise HyTypeError(
-                expr,
+            raise self._syntax_error(expr,
                 "`try' cannot have `else' without `except'")
         # Likewise a bare (try) or (try BODY).
         if not (handlers or finalbody):
-            raise HyTypeError(
-                expr,
+            raise self._syntax_error(expr,
                 "`try' must have an `except' or `finally' clause")
 
         returnable = Result(
@@ -963,7 +963,8 @@ class HyASTCompiler(object):
     def compile_decorate_expression(self, expr, name, args):
         decs, fn = args[:-1], self.compile(args[-1])
         if not fn.stmts or not isinstance(fn.stmts[-1], _decoratables):
-            raise HyTypeError(args[-1], "Decorated a non-function")
+            raise self._syntax_error(args[-1],
+                "Decorated a non-function")
         decs, ret, _ = self._compile_collect(decs)
         fn.stmts[-1].decorator_list = decs + fn.stmts[-1].decorator_list
         return ret + fn
@@ -1194,8 +1195,8 @@ class HyASTCompiler(object):
                 if (HySymbol('*'), None) in kids:
                     if len(kids) != 1:
                         star = kids[kids.index((HySymbol('*'), None))][0]
-                        raise HyTypeError(star, "* in an import name list "
-                                                "must be on its own")
+                        raise self._syntax_error(star,
+                            "* in an import name list must be on its own")
                 else:
                     assignments = [(k, v or k) for k, v in kids]
 
@@ -1390,15 +1391,15 @@ class HyASTCompiler(object):
         if str_name in (["None"] + (["True", "False"] if PY3 else [])):
             # Python 2 allows assigning to True and False, although
             # this is rarely wise.
-            raise HyTypeError(name,
-                              "Can't assign to `%s'" % str_name)
+            raise self._syntax_error(name,
+                "Can't assign to `%s'" % str_name)
 
         result = self.compile(result)
         ld_name = self.compile(name)
 
         if isinstance(ld_name.expr, ast.Call):
-            raise HyTypeError(name,
-                              "Can't assign to a callable: `%s'" % str_name)
+            raise self._syntax_error(name,
+                "Can't assign to a callable: `%s'" % str_name)
 
         if (result.temp_variables
                 and isinstance(name, HySymbol)
@@ -1474,7 +1475,8 @@ class HyASTCompiler(object):
         mandatory, optional, rest, kwonly, kwargs = params
         optional, defaults, ret = self._parse_optional_args(optional)
         if kwonly is not None and not PY3:
-            raise HyTypeError(params, "&kwonly parameters require Python 3")
+            raise self._syntax_error(params,
+                "&kwonly parameters require Python 3")
         kwonly, kw_defaults, ret2 = self._parse_optional_args(kwonly, True)
         ret += ret2
         main_args = mandatory + optional
@@ -1627,8 +1629,8 @@ class HyASTCompiler(object):
             return self.compile(expr)
 
         if not expr:
-            raise HyTypeError(
-                expr, "empty expressions are not allowed at top level")
+            raise self._syntax_error(expr,
+                "empty expressions are not allowed at top level")
 
         args = list(expr)
         root = args.pop(0)
@@ -1646,8 +1648,7 @@ class HyASTCompiler(object):
                     sroot in (mangle(","), mangle(".")) or
                     not any(is_unpack("iterable", x) for x in args)):
                 if sroot in _bad_roots:
-                    raise HyTypeError(
-                        expr,
+                    raise self._syntax_error(expr,
                         "The special form '{}' is not allowed here".format(root))
                 # `sroot` is a special operator. Get the build method and
                 # pattern-match the arguments.
@@ -1655,11 +1656,10 @@ class HyASTCompiler(object):
                 try:
                     parse_tree = pattern.parse(args)
                 except NoParseError as e:
-                    raise HyTypeError(
+                    raise self._syntax_error(
                         expr[min(e.state.pos + 1, len(expr) - 1)],
                         "parse error for special form '{}': {}".format(
-                            root,
-                            e.msg.replace("<EOF>", "end of form")))
+                            root, e.msg.replace("<EOF>", "end of form")))
                 return Result() + build_method(
                     self, expr, unmangle(sroot), *parse_tree)
 
@@ -1681,13 +1681,13 @@ class HyASTCompiler(object):
                         FORM +
                         many(FORM)).parse(args)
                 except NoParseError:
-                    raise HyTypeError(
-                        expr, "attribute access requires object")
+                    raise self._syntax_error(expr,
+                        "attribute access requires object")
                 # Reconstruct `args` to exclude `obj`.
                 args = [x for p in kws for x in p] + list(rest)
                 if is_unpack("iterable", obj):
-                    raise HyTypeError(
-                        obj, "can't call a method on an unpacking form")
+                    raise self._syntax_error(obj,
+                        "can't call a method on an unpacking form")
                 func = self.compile(HyExpression(
                     [HySymbol(".").replace(root), obj] +
                     attrs))
@@ -1725,16 +1725,12 @@ class HyASTCompiler(object):
             glob, local = symbol.rsplit(".", 1)
 
             if not glob:
-                raise HyTypeError(symbol, 'cannot access attribute on '
-                                          'anything other than a name '
-                                          '(in order to get attributes of '
-                                          'expressions, use '
-                                          '`(. <expression> {attr})` or '
-                                          '`(.{attr} <expression>)`)'.format(
-                                              attr=local))
+                raise self._syntax_error(symbol,
+                    'cannot access attribute on anything other than a name (in order to get attributes of expressions, use `(. <expression> {attr})` or `(.{attr} <expression>)`)'.format(attr=local))
 
             if not local:
-                raise HyTypeError(symbol, 'cannot access empty attribute')
+                raise self._syntax_error(symbol,
+                    'cannot access empty attribute')
 
             glob = HySymbol(glob).replace(symbol)
             ret = self.compile_symbol(glob)

From 51c7efe6e8f3b69c9d0eb1a316e11ec228cfd159 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sun, 28 Oct 2018 20:43:17 -0500
Subject: [PATCH 100/223] Retain compiled source and file information for
 exceptions

This commit refactors the exception/error classes and their handling.
It also retains Hy source strings and their originating file information, when
available, all throughout the core parser and compiler functions.

As well, with these changes, calling code is no longer responsible for providing
source and file details to exceptions,

Closes hylang/hy#657.
---
 hy/_compat.py                   |  36 ++++++-
 hy/cmdline.py                   |  46 +++++----
 hy/compiler.py                  | 170 ++++++++++++++++++++++++-------
 hy/core/bootstrap.hy            |  20 ++--
 hy/errors.py                    | 174 +++++++++++++++++++++++++-------
 hy/importer.py                  |  39 +++----
 hy/lex/__init__.py              |  51 +++++++---
 hy/lex/exceptions.py            |  59 ++++-------
 hy/lex/parser.py                | 103 +++++++++----------
 hy/macros.py                    |  50 +++++----
 tests/compilers/test_ast.py     |   6 +-
 tests/importer/test_importer.py |  25 ++---
 tests/test_bin.py               |   2 +-
 tests/test_lex.py               |  22 +++-
 14 files changed, 525 insertions(+), 278 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index bd9390f..55180ca 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -6,7 +6,7 @@ try:
     import __builtin__ as builtins
 except ImportError:
     import builtins  # NOQA
-import sys, keyword
+import sys, keyword, textwrap
 
 PY3 = sys.version_info[0] >= 3
 PY35 = sys.version_info >= (3, 5)
@@ -22,11 +22,39 @@ bytes_type   = bytes if PY3 else str          # NOQA
 long_type    = int   if PY3 else long         # NOQA
 string_types = str   if PY3 else basestring   # NOQA
 
+#
+# Inspired by the same-named `six` functions.
+#
 if PY3:
-    exec('def raise_empty(t, *args): raise t(*args) from None')
+    raise_src = textwrap.dedent('''
+    def raise_from(value, from_value):
+        try:
+            raise value from from_value
+        finally:
+            traceback = None
+    ''')
+
+    def reraise(exc_type, value, traceback=None):
+        try:
+            raise value.with_traceback(traceback)
+        finally:
+            traceback = None
+
 else:
-    def raise_empty(t, *args):
-        raise t(*args)
+    def raise_from(value, from_value=None):
+        raise value
+
+    raise_src = textwrap.dedent('''
+    def reraise(exc_type, value, traceback=None):
+        try:
+            raise exc_type, value, traceback
+        finally:
+            traceback = None
+    ''')
+
+raise_code = compile(raise_src, __file__, 'exec')
+exec(raise_code)
+
 
 def isidentifier(x):
     if x in ('True', 'False', 'None', 'print'):
diff --git a/hy/cmdline.py b/hy/cmdline.py
index 4c2e1c3..70d7b72 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -19,9 +19,9 @@ import astor.code_gen
 
 import hy
 from hy.lex import hy_parse, mangle
-from hy.lex.exceptions import LexException, PrematureEndOfInput
+from hy.lex.exceptions import PrematureEndOfInput
 from hy.compiler import HyASTCompiler, hy_compile, hy_eval
-from hy.errors import HyTypeError
+from hy.errors import HyTypeError, HyLanguageError, HySyntaxError
 from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
@@ -101,15 +101,11 @@ class HyREPL(code.InteractiveConsole, object):
                 self.showtraceback()
 
         try:
-            try:
-                do = hy_parse(source)
-            except PrematureEndOfInput:
-                return True
-        except LexException as e:
-            if e.source is None:
-                e.source = source
-                e.filename = filename
-            error_handler(e, use_simple_traceback=True)
+            do = hy_parse(source, filename=filename)
+        except PrematureEndOfInput:
+            return True
+        except HySyntaxError as e:
+            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
             return False
 
         try:
@@ -121,9 +117,12 @@ class HyREPL(code.InteractiveConsole, object):
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
 
-            value = hy_eval(do, self.locals,
+            value = hy_eval(do, self.locals, self.module,
                             ast_callback=ast_callback,
-                            compiler=self.hy_compiler)
+                            compiler=self.hy_compiler,
+                            filename=filename,
+                            source=source)
+
         except HyTypeError as e:
             if e.source is None:
                 e.source = source
@@ -131,7 +130,7 @@ class HyREPL(code.InteractiveConsole, object):
             error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
             return False
         except Exception as e:
-            error_handler(e)
+            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
             return False
 
         if value is not None:
@@ -208,17 +207,19 @@ SIMPLE_TRACEBACKS = True
 def pretty_error(func, *args, **kw):
     try:
         return func(*args, **kw)
-    except (HyTypeError, LexException) as e:
+    except HyLanguageError as e:
         if SIMPLE_TRACEBACKS:
             print(e, file=sys.stderr)
             sys.exit(1)
         raise
 
 
-def run_command(source):
-    tree = hy_parse(source)
-    require("hy.cmdline", "__main__", assignments="ALL")
-    pretty_error(hy_eval, tree, None, importlib.import_module('__main__'))
+def run_command(source, filename=None):
+    tree = hy_parse(source, filename=filename)
+    __main__ = importlib.import_module('__main__')
+    require("hy.cmdline", __main__, assignments="ALL")
+    pretty_error(hy_eval, tree, None, __main__, filename=filename,
+                 source=source)
     return 0
 
 
@@ -340,7 +341,7 @@ def cmdline_handler(scriptname, argv):
 
     if options.command:
         # User did "hy -c ..."
-        return run_command(options.command)
+        return run_command(options.command, filename='<string>')
 
     if options.mod:
         # User did "hy -m ..."
@@ -356,7 +357,7 @@ def cmdline_handler(scriptname, argv):
     if options.args:
         if options.args[0] == "-":
             # Read the program from stdin
-            return run_command(sys.stdin.read())
+            return run_command(sys.stdin.read(), filename='<stdin>')
 
         else:
             # User did "hy <filename>"
@@ -447,11 +448,12 @@ def hy2py_main():
 
     if options.FILE is None or options.FILE == '-':
         source = sys.stdin.read()
+        hst = pretty_error(hy_parse, source, filename='<stdin>')
     else:
         with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
             source = source_file.read()
+            hst = hy_parse(source, filename=options.FILE)
 
-    hst = pretty_error(hy_parse, source)
     if options.with_source:
         # need special printing on Windows in case the
         # codepage doesn't support utf-8 characters
diff --git a/hy/compiler.py b/hy/compiler.py
index 586129e..5377c33 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -9,20 +9,21 @@ from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
 from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole,
                                notpexpr, dolike, pexpr, times, Tag, tag, unpack)
 from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
-from hy.errors import HyCompileError, HyTypeError
+from hy.errors import (HyCompileError, HyTypeError, HyEvalError,
+                       HyInternalError)
 
 from hy.lex import mangle, unmangle
 
-from hy._compat import (str_type, string_types, bytes_type, long_type, PY3,
-                        PY35, raise_empty)
+from hy._compat import (string_types, str_type, bytes_type, long_type, PY3,
+                        PY35, reraise)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
 import hy.core
 
+import pkgutil
 import traceback
 import importlib
 import inspect
-import pkgutil
 import types
 import ast
 import sys
@@ -340,22 +341,33 @@ def is_unpack(kind, x):
 class HyASTCompiler(object):
     """A Hy-to-Python AST compiler"""
 
-    def __init__(self, module):
+    def __init__(self, module, filename=None, source=None):
         """
         Parameters
         ----------
         module: str or types.ModuleType
-            Module in which the Hy tree is evaluated.
+            Module name or object in which the Hy tree is evaluated.
+        filename: str, optional
+            The name of the file for the source to be compiled.
+            This is optional information for informative error messages and
+            debugging.
+        source: str, optional
+            The source for the file, if any, being compiled.  This is optional
+            information for informative error messages and debugging.
         """
         self.anon_var_count = 0
         self.imports = defaultdict(set)
         self.temp_if = None
 
         if not inspect.ismodule(module):
-            module = importlib.import_module(module)
+            self.module = importlib.import_module(module)
+        else:
+            self.module = module
 
-        self.module = module
-        self.module_name = module.__name__
+        self.module_name = self.module.__name__
+
+        self.filename = filename
+        self.source = source
 
         # Hy expects these to be present, so we prep the module for Hy
         # compilation.
@@ -431,13 +443,15 @@ class HyASTCompiler(object):
             # nested; so let's re-raise this exception, let's not wrap it in
             # another HyCompileError!
             raise
-        except HyTypeError:
-            raise
+        except HyTypeError as e:
+            reraise(type(e), e, None)
         except Exception as e:
-            raise_empty(HyCompileError, e, sys.exc_info()[2])
+            f_exc = traceback.format_exc()
+            exc_msg = "Internal Compiler Bug 😱\n⤷ {}".format(f_exc)
+            reraise(HyCompileError, HyCompileError(exc_msg), sys.exc_info()[2])
 
     def _syntax_error(self, expr, message):
-        return HyTypeError(expr, message)
+        return HyTypeError(message, self.filename, expr, self.source)
 
     def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
                          oldpy_unpack=False):
@@ -1614,7 +1628,29 @@ class HyASTCompiler(object):
     def compile_eval_and_compile(self, expr, root, body):
         new_expr = HyExpression([HySymbol("do").replace(expr[0])]).replace(expr)
 
-        hy_eval(new_expr + body, self.module.__dict__, self.module)
+        try:
+            hy_eval(new_expr + body,
+                    self.module.__dict__,
+                    self.module,
+                    filename=self.filename,
+                    source=self.source)
+        except HyInternalError:
+            # Unexpected "meta" compilation errors need to be treated
+            # like normal (unexpected) compilation errors at this level
+            # (or the compilation level preceding this one).
+            raise
+        except Exception as e:
+            # These could be expected Hy language errors (e.g. syntax errors)
+            # or regular Python runtime errors that do not signify errors in
+            # the compilation *process* (although compilation did technically
+            # fail).
+            # We wrap these exceptions and pass them through.
+            reraise(HyEvalError,
+                    HyEvalError(str(e),
+                                self.filename,
+                                body,
+                                self.source),
+                    sys.exc_info()[2])
 
         return (self._compile_branch(body)
                 if ast_str(root) == "eval_and_compile"
@@ -1798,8 +1834,13 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False):
 
 
 def hy_eval(hytree, locals=None, module=None, ast_callback=None,
-            compiler=None):
+            compiler=None, filename='<string>', source=None):
     """Evaluates a quoted expression and returns the value.
+
+    If you're evaluating hand-crafted AST trees, make sure the line numbers
+    are set properly.  Try `fix_missing_locations` and related functions in the
+    Python `ast` library.
+
     Examples
     --------
        => (eval '(print "Hello World"))
@@ -1812,8 +1853,8 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None,
 
     Parameters
     ----------
-    hytree: a Hy expression tree
-        Source code to parse.
+    hytree: HyObject
+        The Hy AST object to evaluate.
 
     locals: dict, optional
         Local environment in which to evaluate the Hy tree.  Defaults to the
@@ -1835,6 +1876,19 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None,
         An existing Hy compiler to use for compilation.  Also serves as
         the `module` value when given.
 
+    filename: str, optional
+        The filename corresponding to the source for `tree`.  This will be
+        overridden by the `filename` field of `tree`, if any; otherwise, it
+        defaults to "<string>".  When `compiler` is given, its `filename` field
+        value is always used.
+
+    source: str, optional
+        A string containing the source code for `tree`.  This will be
+        overridden by the `source` field of `tree`, if any; otherwise,
+        if `None`, an attempt will be made to obtain it from the module given by
+        `module`.  When `compiler` is given, its `source` field value is always
+        used.
+
     Returns
     -------
     out : Result of evaluating the Hy compiled tree.
@@ -1849,36 +1903,53 @@ def hy_eval(hytree, locals=None, module=None, ast_callback=None,
     if not isinstance(locals, dict):
         raise TypeError("Locals must be a dictionary")
 
-    _ast, expr = hy_compile(hytree, module=module, get_expr=True,
-                            compiler=compiler)
+    # Does the Hy AST object come with its own information?
+    filename = getattr(hytree, 'filename', filename) or '<string>'
+    source = getattr(hytree, 'source', source)
 
-    # Spoof the positions in the generated ast...
-    for node in ast.walk(_ast):
-        node.lineno = 1
-        node.col_offset = 1
-
-    for node in ast.walk(expr):
-        node.lineno = 1
-        node.col_offset = 1
+    _ast, expr = hy_compile(hytree, module, get_expr=True,
+                            compiler=compiler, filename=filename,
+                            source=source)
 
     if ast_callback:
         ast_callback(_ast, expr)
 
-    globals = module.__dict__
-
     # Two-step eval: eval() the body of the exec call
-    eval(ast_compile(_ast, "<eval_body>", "exec"), globals, locals)
+    eval(ast_compile(_ast, filename, "exec"),
+         module.__dict__, locals)
 
     # Then eval the expression context and return that
-    return eval(ast_compile(expr, "<eval>", "eval"), globals, locals)
+    return eval(ast_compile(expr, filename, "eval"),
+                module.__dict__, locals)
 
 
-def hy_compile(tree, module=None, root=ast.Module, get_expr=False,
-               compiler=None):
-    """Compile a Hy tree into a Python AST tree.
+def _module_file_source(module_name, filename, source):
+    """Try to obtain missing filename and source information from a module name
+    without actually loading the module.
+    """
+    if filename is None or source is None:
+        mod_loader = pkgutil.get_loader(module_name)
+        if mod_loader:
+            if filename is None:
+                filename = mod_loader.get_filename(module_name)
+            if source is None:
+                source = mod_loader.get_source(module_name)
+
+    # We need a non-None filename.
+    filename = filename or '<string>'
+
+    return filename, source
+
+
+def hy_compile(tree, module, root=ast.Module, get_expr=False,
+               compiler=None, filename=None, source=None):
+    """Compile a HyObject tree into a Python AST Module.
 
     Parameters
     ----------
+    tree: HyObject
+        The Hy AST object to compile.
+
     module: str or types.ModuleType, optional
         Module, or name of the module, in which the Hy tree is evaluated.
         The module associated with `compiler` takes priority over this value.
@@ -1893,18 +1964,43 @@ def hy_compile(tree, module=None, root=ast.Module, get_expr=False,
         An existing Hy compiler to use for compilation.  Also serves as
         the `module` value when given.
 
+    filename: str, optional
+        The filename corresponding to the source for `tree`.  This will be
+        overridden by the `filename` field of `tree`, if any; otherwise, it
+        defaults to "<string>".  When `compiler` is given, its `filename` field
+        value is always used.
+
+    source: str, optional
+        A string containing the source code for `tree`.  This will be
+        overridden by the `source` field of `tree`, if any; otherwise,
+        if `None`, an attempt will be made to obtain it from the module given by
+        `module`.  When `compiler` is given, its `source` field value is always
+        used.
+
     Returns
     -------
     out : A Python AST tree
     """
     module = get_compiler_module(module, compiler, False)
 
+    if isinstance(module, string_types):
+        if module.startswith('<') and module.endswith('>'):
+            module = types.ModuleType(module)
+        else:
+            module = importlib.import_module(ast_str(module, piecewise=True))
+
+    if not inspect.ismodule(module):
+        raise TypeError('Invalid module type: {}'.format(type(module)))
+
+    filename = getattr(tree, 'filename', filename)
+    source = getattr(tree, 'source', source)
+
     tree = wrap_value(tree)
     if not isinstance(tree, HyObject):
-        raise HyCompileError("`tree` must be a HyObject or capable of "
-                             "being promoted to one")
+        raise TypeError("`tree` must be a HyObject or capable of "
+                        "being promoted to one")
 
-    compiler = compiler or HyASTCompiler(module)
+    compiler = compiler or HyASTCompiler(module, filename=filename, source=source)
     result = compiler.compile(tree)
     expr = result.force_expr
 
diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy
index fa613a6..0c98f60 100644
--- a/hy/core/bootstrap.hy
+++ b/hy/core/bootstrap.hy
@@ -14,15 +14,14 @@
      (if* (not (isinstance macro-name hy.models.HySymbol))
           (raise
             (hy.errors.HyTypeError
-              macro-name
               (% "received a `%s' instead of a symbol for macro name"
-                 (. (type name)
-                    __name__)))))
+                 (. (type name) --name--))
+              --file-- macro-name None)))
      (for [kw '[&kwonly &kwargs]]
        (if* (in kw lambda-list)
-            (raise (hy.errors.HyTypeError macro-name
-                                          (% "macros cannot use %s"
-                                             kw)))))
+            (raise (hy.errors.HyTypeError (% "macros cannot use %s"
+                                             kw)
+                                          --file-- macro-name None))))
      ;; this looks familiar...
      `(eval-and-compile
         (import hy)
@@ -45,9 +44,9 @@
   (if (and (not (isinstance tag-name hy.models.HySymbol))
            (not (isinstance tag-name hy.models.HyString)))
       (raise (hy.errors.HyTypeError
-               tag-name
                (% "received a `%s' instead of a symbol for tag macro name"
-                  (. (type tag-name) __name__)))))
+                  (. (type tag-name) --name--))
+               --file-- tag-name None)))
   (if (or (= tag-name ":")
           (= tag-name "&"))
       (raise (NameError (% "%s can't be used as a tag macro name" tag-name))))
@@ -58,9 +57,8 @@
      ((hy.macros.tag ~tag-name)
       (fn ~lambda-list ~@body))))
 
-(defmacro macro-error [location reason]
-  "Error out properly within a macro at `location` giving `reason`."
-  `(raise (hy.errors.HyMacroExpansionError ~location ~reason)))
+(defmacro macro-error [expression reason &optional [filename '--name--]]
+  `(raise (hy.errors.HyMacroExpansionError ~reason ~filename ~expression None)))
 
 (defmacro defn [name lambda-list &rest body]
   "Define `name` as a function with `lambda-list` signature and body `body`."
diff --git a/hy/errors.py b/hy/errors.py
index 7d36ab2..e8e9f98 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -5,41 +5,65 @@
 
 import traceback
 
+from functools import reduce
+
 from clint.textui import colored
 
 
 class HyError(Exception):
-    """
-    Generic Hy error. All internal Exceptions will be subclassed from this
-    Exception.
-    """
-    pass
-
-
-class HyCompileError(HyError):
-    def __init__(self, exception, traceback=None):
-        self.exception = exception
-        self.traceback = traceback
-
-    def __str__(self):
-        if isinstance(self.exception, HyTypeError):
-            return str(self.exception)
-        if self.traceback:
-            tb = "".join(traceback.format_tb(self.traceback)).strip()
-        else:
-            tb = "No traceback available. 😟"
-        return("Internal Compiler Bug 😱\n⤷ %s: %s\nCompilation traceback:\n%s"
-               % (self.exception.__class__.__name__,
-                  self.exception, tb))
-
-
-class HyTypeError(TypeError):
-    def __init__(self, expression, message):
-        super(HyTypeError, self).__init__(message)
-        self.expression = expression
+    def __init__(self, message, *args):
         self.message = message
-        self.source = None
-        self.filename = None
+        super(HyError, self).__init__(message, *args)
+
+
+class HyInternalError(HyError):
+    """Unexpected errors occurring during compilation or parsing of Hy code.
+
+    Errors sub-classing this are not intended to be user-facing, and will,
+    hopefully, never be seen by users!
+    """
+
+    def __init__(self, message, *args):
+        super(HyInternalError, self).__init__(message, *args)
+
+
+class HyLanguageError(HyError):
+    """Errors caused by invalid use of the Hy language.
+
+    This, and any errors inheriting from this, are user-facing.
+    """
+
+    def __init__(self, message, *args):
+        super(HyLanguageError, self).__init__(message, *args)
+
+
+class HyCompileError(HyInternalError):
+    """Unexpected errors occurring within the compiler."""
+
+
+class HyTypeError(HyLanguageError, TypeError):
+    """TypeErrors occurring during the normal use of Hy."""
+
+    def __init__(self, message, filename=None, expression=None, source=None):
+        """
+        Parameters
+        ----------
+        message: str
+            The message to display for this error.
+        filename: str, optional
+            The filename for the source code generating this error.
+        expression: HyObject, optional
+            The Hy expression generating this error.
+        source: str, optional
+            The actual source code generating this error.
+        """
+        self.message = message
+        self.filename = filename
+        self.expression = expression
+        self.source = source
+
+        super(HyTypeError, self).__init__(message, filename, expression,
+                                          source)
 
     def __str__(self):
 
@@ -93,12 +117,92 @@ class HyTypeError(TypeError):
 
 
 class HyMacroExpansionError(HyTypeError):
-    pass
+    """Errors caused by invalid use of Hy macros.
+
+    This, and any errors inheriting from this, are user-facing.
+    """
 
 
-class HyIOError(HyError, IOError):
+class HyEvalError(HyLanguageError):
+    """Errors occurring during code evaluation at compile-time.
+
+    These errors distinguish unexpected errors within the compilation process
+    (i.e. `HyInternalError`s) from unrelated errors in user code evaluated by
+    the compiler (e.g. in `eval-and-compile`).
+
+    This, and any errors inheriting from this, are user-facing.
     """
-    Trivial subclass of IOError and HyError, to distinguish between
-    IOErrors raised by Hy itself as opposed to Hy programs.
+
+
+class HyIOError(HyInternalError, IOError):
+    """ Subclass used to distinguish between IOErrors raised by Hy itself as
+    opposed to Hy programs.
     """
-    pass
+
+
+class HySyntaxError(HyLanguageError, SyntaxError):
+    """Error during the Lexing of a Hython expression."""
+
+    def __init__(self, message, filename=None, lineno=-1, colno=-1,
+                 source=None):
+        """
+        Parameters
+        ----------
+        message: str
+            The exception's message.
+        filename: str, optional
+            The filename for the source code generating this error.
+        lineno: int, optional
+            The line number of the error.
+        colno: int, optional
+            The column number of the error.
+        source: str, optional
+            The actual source code generating this error.
+        """
+        self.message = message
+        self.filename = filename
+        self.lineno = lineno
+        self.colno = colno
+        self.source = source
+        super(HySyntaxError, self).__init__(message,
+                                            # The builtin `SyntaxError` needs a
+                                            # tuple.
+                                            (filename, lineno, colno, source))
+
+    @staticmethod
+    def from_expression(message, expression, filename=None, source=None):
+        if not source:
+            # Maybe the expression object has its own source.
+            source = getattr(expression, 'source', None)
+
+        if not filename:
+            filename = getattr(expression, 'filename', None)
+
+        if source:
+            lineno = expression.start_line
+            colno = expression.start_column
+            end_line = getattr(expression, 'end_line', len(source))
+            lines = source.splitlines()
+            source = '\n'.join(lines[lineno-1:end_line])
+        else:
+            # We could attempt to extract the source given a filename, but we
+            # don't.
+            lineno = colno = -1
+
+        return HySyntaxError(message, filename, lineno, colno, source)
+
+    def __str__(self):
+
+        output = traceback.format_exception_only(SyntaxError, self)
+
+        output[-1] = colored.yellow(output[-1])
+        if len(self.source) > 0:
+            output[-2] = colored.green(output[-2])
+            for line in output[::-2]:
+                if line.strip().startswith(
+                        'File "{}", line'.format(self.filename)):
+                    break
+            output[-3] = colored.red(output[-3])
+
+        # Avoid "...expected str instance, ColoredString found"
+        return reduce(lambda x, y: x + y, output)
diff --git a/hy/importer.py b/hy/importer.py
index e3cc50d..0e59498 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -17,10 +17,8 @@ import importlib
 from functools import partial
 from contextlib import contextmanager
 
-from hy.errors import HyTypeError
 from hy.compiler import hy_compile, hy_ast_compile_flags
 from hy.lex import hy_parse
-from hy.lex.exceptions import LexException
 from hy._compat import PY3
 
 
@@ -153,15 +151,9 @@ if PY3:
     def _hy_source_to_code(self, data, path, _optimize=-1):
         if _could_be_hy_src(path):
             source = data.decode("utf-8")
-            try:
-                hy_tree = hy_parse(source)
-                with loader_module_obj(self) as module:
-                    data = hy_compile(hy_tree, module)
-            except (HyTypeError, LexException) as e:
-                if e.source is None:
-                    e.source = source
-                    e.filename = path
-                raise
+            hy_tree = hy_parse(source, filename=path)
+            with loader_module_obj(self) as module:
+                data = hy_compile(hy_tree, module)
 
         return _py_source_to_code(self, data, path, _optimize=_optimize)
 
@@ -287,19 +279,15 @@ else:
             fullname = self._fix_name(fullname)
             if fullname is None:
                 fullname = self.fullname
-            try:
-                hy_source = self.get_source(fullname)
-                hy_tree = hy_parse(hy_source)
-                with loader_module_obj(self) as module:
-                    hy_ast = hy_compile(hy_tree, module)
 
-                code = compile(hy_ast, self.filename, 'exec',
-                               hy_ast_compile_flags)
-            except (HyTypeError, LexException) as e:
-                if e.source is None:
-                    e.source = hy_source
-                    e.filename = self.filename
-                raise
+            hy_source = self.get_source(fullname)
+            hy_tree = hy_parse(hy_source, filename=self.filename)
+
+            with loader_module_obj(self) as module:
+                hy_ast = hy_compile(hy_tree, module)
+
+            code = compile(hy_ast, self.filename, 'exec',
+                           hy_ast_compile_flags)
 
             if not sys.dont_write_bytecode:
                 try:
@@ -453,7 +441,7 @@ else:
             try:
                 flags = None
                 if _could_be_hy_src(filename):
-                    hy_tree = hy_parse(source_str)
+                    hy_tree = hy_parse(source_str, filename=filename)
 
                     if module is None:
                         module = inspect.getmodule(inspect.stack()[1][0])
@@ -465,9 +453,6 @@ else:
 
                 codeobject = compile(source, dfile or filename, 'exec', flags)
             except Exception as err:
-                if isinstance(err, (HyTypeError, LexException)) and err.source is None:
-                    err.source = source_str
-                    err.filename = filename
 
                 py_exc = py_compile.PyCompileError(err.__class__, err,
                                                    dfile or filename)
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index bf82cf9..f1465cd 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -8,9 +8,10 @@ import re
 import sys
 import unicodedata
 
-from hy._compat import str_type, isidentifier, UCS4
+from hy._compat import str_type, isidentifier, UCS4, reraise
 from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
+from hy.errors import HySyntaxError
 
 try:
     from io import StringIO
@@ -18,7 +19,7 @@ except ImportError:
     from StringIO import StringIO
 
 
-def hy_parse(source):
+def hy_parse(source, filename='<string>'):
     """Parse a Hy source string.
 
     Parameters
@@ -26,31 +27,51 @@ def hy_parse(source):
     source: string
         Source code to parse.
 
+    filename: string, optional
+        File name corresponding to source.  Defaults to "<string>".
+
     Returns
     -------
-    out : instance of `types.CodeType`
+    out : HyExpression
     """
-    source = re.sub(r'\A#!.*', '', source)
-    return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
+    _source = re.sub(r'\A#!.*', '', source)
+    try:
+        res = HyExpression([HySymbol("do")] +
+                           tokenize(_source + "\n",
+                                    filename=filename))
+        res.source = source
+        res.filename = filename
+        return res
+    except HySyntaxError as e:
+        reraise(type(e), e, None)
 
 
-def tokenize(buf):
-    """
-    Tokenize a Lisp file or string buffer into internal Hy objects.
+class ParserState(object):
+    def __init__(self, source, filename):
+        self.source = source
+        self.filename = filename
+
+
+def tokenize(source, filename=None):
+    """ Tokenize a Lisp file or string buffer into internal Hy objects.
+
+    Parameters
+    ----------
+    source: str
+        The source to tokenize.
+    filename: str, optional
+        The filename corresponding to `source`.
     """
     from hy.lex.lexer import lexer
     from hy.lex.parser import parser
     from rply.errors import LexingError
     try:
-        return parser.parse(lexer.lex(buf))
+        return parser.parse(lexer.lex(source),
+                            state=ParserState(source, filename))
     except LexingError as e:
         pos = e.getsourcepos()
-        raise LexException("Could not identify the next token.",
-                           pos.lineno, pos.colno, buf)
-    except LexException as e:
-        if e.source is None:
-            e.source = buf
-        raise
+        raise LexException("Could not identify the next token.", filename,
+                           pos.lineno, pos.colno, source)
 
 
 mangle_delim = 'X'
diff --git a/hy/lex/exceptions.py b/hy/lex/exceptions.py
index 573a8e8..fb9aa7a 100644
--- a/hy/lex/exceptions.py
+++ b/hy/lex/exceptions.py
@@ -1,49 +1,34 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
-
-from hy.errors import HyError
+from hy.errors import HySyntaxError
 
 
-class LexException(HyError):
-    """Error during the Lexing of a Hython expression."""
-    def __init__(self, message, lineno, colno, source=None):
-        super(LexException, self).__init__(message)
-        self.message = message
-        self.lineno = lineno
-        self.colno = colno
-        self.source = source
-        self.filename = '<stdin>'
+class LexException(HySyntaxError):
 
-    def __str__(self):
-        from hy.errors import colored
+    @classmethod
+    def from_lexer(cls, message, state, token):
+        source_pos = token.getsourcepos()
+        if token.source_pos:
+            lineno = source_pos.lineno
+            colno = source_pos.colno
+        else:
+            lineno = -1
+            colno = -1
 
-        line = self.lineno
-        start = self.colno
+        if state.source:
+            lines = state.source.splitlines()
+            if lines[-1] == '':
+                del lines[-1]
 
-        result = ""
+            if lineno < 1:
+                lineno = len(lines)
+            if colno < 1:
+                colno = len(lines[-1])
 
-        source = self.source.split("\n")
-
-        if line > 0 and start > 0:
-            result += '  File "%s", line %d, column %d\n\n' % (self.filename,
-                                                               line,
-                                                               start)
-
-            if len(self.source) > 0:
-                source_line = source[line-1]
-            else:
-                source_line = ""
-
-            result += '  %s\n' % colored.red(source_line)
-            result += '  %s%s\n' % (' '*(start-1), colored.green('^'))
-
-        result += colored.yellow("LexException: %s\n\n" % self.message)
-
-        return result
+            source = lines[lineno - 1]
+        return cls(message, state.filename, lineno, colno, source)
 
 
 class PrematureEndOfInput(LexException):
-    """We got a premature end of input"""
-    def __init__(self, message):
-        super(PrematureEndOfInput, self).__init__(message, -1, -1)
+    pass
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index c602734..f5cd5e5 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -22,10 +22,10 @@ pg = ParserGenerator([rule.name for rule in lexer.rules] + ['$end'])
 
 def set_boundaries(fun):
     @wraps(fun)
-    def wrapped(p):
+    def wrapped(state, p):
         start = p[0].source_pos
         end = p[-1].source_pos
-        ret = fun(p)
+        ret = fun(state, p)
         ret.start_line = start.lineno
         ret.start_column = start.colno
         if start is not end:
@@ -40,9 +40,9 @@ def set_boundaries(fun):
 
 def set_quote_boundaries(fun):
     @wraps(fun)
-    def wrapped(p):
+    def wrapped(state, p):
         start = p[0].source_pos
-        ret = fun(p)
+        ret = fun(state, p)
         ret.start_line = start.lineno
         ret.start_column = start.colno
         ret.end_line = p[-1].end_line
@@ -52,54 +52,45 @@ def set_quote_boundaries(fun):
 
 
 @pg.production("main : list_contents")
-def main(p):
+def main(state, p):
     return p[0]
 
 
 @pg.production("main : $end")
-def main_empty(p):
+def main_empty(state, p):
     return []
 
 
-def reject_spurious_dots(*items):
-    "Reject the spurious dots from items"
-    for list in items:
-        for tok in list:
-            if tok == "." and type(tok) == HySymbol:
-                raise LexException("Malformed dotted list",
-                                   tok.start_line, tok.start_column)
-
-
 @pg.production("paren : LPAREN list_contents RPAREN")
 @set_boundaries
-def paren(p):
+def paren(state, p):
     return HyExpression(p[1])
 
 
 @pg.production("paren : LPAREN RPAREN")
 @set_boundaries
-def empty_paren(p):
+def empty_paren(state, p):
     return HyExpression([])
 
 
 @pg.production("list_contents : term list_contents")
-def list_contents(p):
+def list_contents(state, p):
     return [p[0]] + p[1]
 
 
 @pg.production("list_contents : term")
-def list_contents_single(p):
+def list_contents_single(state, p):
     return [p[0]]
 
 
 @pg.production("list_contents : DISCARD term discarded_list_contents")
-def list_contents_empty(p):
+def list_contents_empty(state, p):
     return []
 
 
 @pg.production("discarded_list_contents : DISCARD term discarded_list_contents")
 @pg.production("discarded_list_contents :")
-def discarded_list_contents(p):
+def discarded_list_contents(state, p):
     pass
 
 
@@ -109,58 +100,58 @@ def discarded_list_contents(p):
 @pg.production("term : list")
 @pg.production("term : set")
 @pg.production("term : string")
-def term(p):
+def term(state, p):
     return p[0]
 
 
 @pg.production("term : DISCARD term term")
-def term_discard(p):
+def term_discard(state, p):
     return p[2]
 
 
 @pg.production("term : QUOTE term")
 @set_quote_boundaries
-def term_quote(p):
+def term_quote(state, p):
     return HyExpression([HySymbol("quote"), p[1]])
 
 
 @pg.production("term : QUASIQUOTE term")
 @set_quote_boundaries
-def term_quasiquote(p):
+def term_quasiquote(state, p):
     return HyExpression([HySymbol("quasiquote"), p[1]])
 
 
 @pg.production("term : UNQUOTE term")
 @set_quote_boundaries
-def term_unquote(p):
+def term_unquote(state, p):
     return HyExpression([HySymbol("unquote"), p[1]])
 
 
 @pg.production("term : UNQUOTESPLICE term")
 @set_quote_boundaries
-def term_unquote_splice(p):
+def term_unquote_splice(state, p):
     return HyExpression([HySymbol("unquote-splice"), p[1]])
 
 
 @pg.production("term : HASHSTARS term")
 @set_quote_boundaries
-def term_hashstars(p):
+def term_hashstars(state, p):
     n_stars = len(p[0].getstr()[1:])
     if n_stars == 1:
         sym = "unpack-iterable"
     elif n_stars == 2:
         sym = "unpack-mapping"
     else:
-        raise LexException(
+        raise LexException.from_lexer(
             "Too many stars in `#*` construct (if you want to unpack a symbol "
             "beginning with a star, separate it with whitespace)",
-            p[0].source_pos.lineno, p[0].source_pos.colno)
+            state, p[0])
     return HyExpression([HySymbol(sym), p[1]])
 
 
 @pg.production("term : HASHOTHER term")
 @set_quote_boundaries
-def hash_other(p):
+def hash_other(state, p):
     # p == [(Token('HASHOTHER', '#foo'), bar)]
     st = p[0].getstr()[1:]
     str_object = HyString(st)
@@ -170,63 +161,63 @@ def hash_other(p):
 
 @pg.production("set : HLCURLY list_contents RCURLY")
 @set_boundaries
-def t_set(p):
+def t_set(state, p):
     return HySet(p[1])
 
 
 @pg.production("set : HLCURLY RCURLY")
 @set_boundaries
-def empty_set(p):
+def empty_set(state, p):
     return HySet([])
 
 
 @pg.production("dict : LCURLY list_contents RCURLY")
 @set_boundaries
-def t_dict(p):
+def t_dict(state, p):
     return HyDict(p[1])
 
 
 @pg.production("dict : LCURLY RCURLY")
 @set_boundaries
-def empty_dict(p):
+def empty_dict(state, p):
     return HyDict([])
 
 
 @pg.production("list : LBRACKET list_contents RBRACKET")
 @set_boundaries
-def t_list(p):
+def t_list(state, p):
     return HyList(p[1])
 
 
 @pg.production("list : LBRACKET RBRACKET")
 @set_boundaries
-def t_empty_list(p):
+def t_empty_list(state, p):
     return HyList([])
 
 
 @pg.production("string : STRING")
 @set_boundaries
-def t_string(p):
+def t_string(state, p):
     # Replace the single double quotes with triple double quotes to allow
     # embedded newlines.
     try:
         s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
     except SyntaxError:
-        raise LexException("Can't convert {} to a HyString".format(p[0].value),
-            p[0].source_pos.lineno, p[0].source_pos.colno)
+        raise LexException.from_lexer("Can't convert {} to a HyString".format(p[0].value),
+                                      state, p[0])
     return (HyString if isinstance(s, str_type) else HyBytes)(s)
 
 
 @pg.production("string : PARTIAL_STRING")
-def t_partial_string(p):
+def t_partial_string(state, p):
     # Any unterminated string requires more input
-    raise PrematureEndOfInput("Premature end of input")
+    raise PrematureEndOfInput.from_lexer("Partial string literal", state, p[0])
 
 
 bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING')
 @pg.production("string : BRACKETSTRING")
 @set_boundaries
-def t_bracket_string(p):
+def t_bracket_string(state, p):
     m = bracket_string_re.match(p[0].value)
     delim, content = m.groups()
     return HyString(content, brackets=delim)
@@ -234,7 +225,7 @@ def t_bracket_string(p):
 
 @pg.production("identifier : IDENTIFIER")
 @set_boundaries
-def t_identifier(p):
+def t_identifier(state, p):
     obj = p[0].value
 
     val = symbol_like(obj)
@@ -243,11 +234,11 @@ def t_identifier(p):
 
     if "." in obj and symbol_like(obj.split(".", 1)[0]) is not None:
         # E.g., `5.attr` or `:foo.attr`
-        raise LexException(
+        raise LexException.from_lexer(
             'Cannot access attribute on anything other than a name (in '
             'order to get attributes of expressions, use '
             '`(. <expression> <attr>)` or `(.<attr> <expression>)`)',
-            p[0].source_pos.lineno, p[0].source_pos.colno)
+            state, p[0])
 
     return HySymbol(obj)
 
@@ -284,14 +275,24 @@ def symbol_like(obj):
 
 
 @pg.error
-def error_handler(token):
+def error_handler(state, token):
     tokentype = token.gettokentype()
     if tokentype == '$end':
-        raise PrematureEndOfInput("Premature end of input")
+        source_pos = token.source_pos or token.getsourcepos()
+        source = state.source
+        if source_pos:
+            lineno = source_pos.lineno
+            colno = source_pos.colno
+        else:
+            lineno = -1
+            colno = -1
+
+        raise PrematureEndOfInput.from_lexer("Premature end of input", state,
+                                             token)
     else:
-        raise LexException(
-            "Ran into a %s where it wasn't expected." % tokentype,
-            token.source_pos.lineno, token.source_pos.colno)
+        raise LexException.from_lexer(
+            "Ran into a %s where it wasn't expected." % tokentype, state,
+            token)
 
 
 parser = pg.build()
diff --git a/hy/macros.py b/hy/macros.py
index be59200..274702f 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -1,14 +1,16 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
+import sys
 import importlib
 import inspect
 import pkgutil
 
-from hy._compat import PY3, string_types
+from contextlib import contextmanager
+
+from hy._compat import PY3, string_types, reraise
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
-
 from hy.errors import HyTypeError, HyMacroExpansionError
 
 try:
@@ -257,6 +259,32 @@ def make_empty_fn_copy(fn):
     return empty_fn
 
 
+@contextmanager
+def macro_exceptions(module, macro_tree, compiler=None):
+    try:
+        yield
+    except Exception as e:
+        try:
+            filename = inspect.getsourcefile(module)
+            source = inspect.getsource(module)
+        except TypeError:
+            if compiler:
+                filename = compiler.filename
+                source = compiler.source
+
+        if not isinstance(e, HyTypeError):
+            exc_type = HyMacroExpansionError
+            msg = "expanding `{}': ".format(macro_tree[0])
+            msg += str(e).replace("<lambda>()", "", 1).strip()
+        else:
+            exc_type = HyTypeError
+            msg = e.message
+
+        reraise(exc_type,
+                exc_type(msg, filename, macro_tree, source),
+                sys.exc_info()[2].tb_next)
+
+
 def macroexpand(tree, module, compiler=None, once=False):
     """Expand the toplevel macros for the given Hy AST tree.
 
@@ -324,23 +352,10 @@ def macroexpand(tree, module, compiler=None, once=False):
                 compiler = HyASTCompiler(module)
             opts['compiler'] = compiler
 
-        try:
+        with macro_exceptions(module, tree, compiler):
             m_copy = make_empty_fn_copy(m)
             m_copy(module.__name__, *tree[1:], **opts)
-        except TypeError as e:
-            msg = "expanding `" + str(tree[0]) + "': "
-            msg += str(e).replace("<lambda>()", "", 1).strip()
-            raise HyMacroExpansionError(tree, msg)
-
-        try:
             obj = m(module.__name__, *tree[1:], **opts)
-        except HyTypeError as e:
-            if e.expression is None:
-                e.expression = tree
-            raise
-        except Exception as e:
-            msg = "expanding `" + str(tree[0]) + "': " + repr(e)
-            raise HyMacroExpansionError(tree, msg)
 
         if isinstance(obj, HyExpression):
             obj.module = inspect.getmodule(m)
@@ -375,7 +390,8 @@ def tag_macroexpand(tag, tree, module):
                      None)
 
     if tag_macro is None:
-        raise HyTypeError(tag, "'{0}' is not a defined tag macro.".format(tag))
+        raise HyTypeError("`{0}' is not a defined tag macro.".format(tag),
+                          None, tag, None)
 
     expr = tag_macro(tree)
 
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 75e9c49..0c7bcd5 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -10,7 +10,7 @@ from hy.models import HyObject
 from hy.compiler import hy_compile, hy_eval
 from hy.errors import HyCompileError, HyTypeError
 from hy.lex import hy_parse
-from hy.lex.exceptions import LexException
+from hy.lex.exceptions import LexException, PrematureEndOfInput
 from hy._compat import PY3
 
 import ast
@@ -474,7 +474,7 @@ def test_lambda_list_keywords_kwonly():
     else:
         exception = cant_compile(kwonly_demo)
         assert isinstance(exception, HyTypeError)
-        message, = exception.args
+        message = exception.args[0]
         assert message == "&kwonly parameters require Python 3"
 
 
@@ -547,7 +547,7 @@ def test_compile_error():
 
 def test_for_compile_error():
     """Ensure we get compile error in tricky 'for' cases"""
-    with pytest.raises(LexException) as excinfo:
+    with pytest.raises(PrematureEndOfInput) as excinfo:
         can_compile("(fn [] (for)")
     assert excinfo.value.message == "Premature end of input"
 
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index 3017d16..dea1baf 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -14,10 +14,10 @@ from fractions import Fraction
 import pytest
 
 import hy
-from hy.errors import HyTypeError
 from hy.lex import hy_parse
-from hy.lex.exceptions import LexException
-from hy.compiler import hy_compile
+from hy.errors import HyLanguageError
+from hy.lex.exceptions import PrematureEndOfInput
+from hy.compiler import hy_eval, hy_compile
 from hy.importer import HyLoader, cache_from_source
 
 try:
@@ -57,7 +57,7 @@ def test_runpy():
 
 
 def test_stringer():
-    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '__main__')
+    _ast = hy_compile(hy_parse("(defn square [x] (* x x))"), __name__)
 
     assert type(_ast.body[0]) == ast.FunctionDef
 
@@ -79,14 +79,8 @@ def test_imports():
 def test_import_error_reporting():
     "Make sure that (import) reports errors correctly."
 
-    def _import_error_test():
-        try:
-            _ = hy_compile(hy_parse("(import \"sys\")"), '__main__')
-        except HyTypeError:
-            return "Error reported"
-
-    assert _import_error_test() == "Error reported"
-    assert _import_error_test() is not None
+    with pytest.raises(HyLanguageError):
+        hy_compile(hy_parse("(import \"sys\")"), __name__)
 
 
 def test_import_error_cleanup():
@@ -124,7 +118,7 @@ def test_import_autocompiles():
 
 def test_eval():
     def eval_str(s):
-        return hy.eval(hy.read_str(s))
+        return hy_eval(hy.read_str(s), filename='<string>', source=s)
 
     assert eval_str('[1 2 3]') == [1, 2, 3]
     assert eval_str('{"dog" "bark" "cat" "meow"}') == {
@@ -205,8 +199,7 @@ def test_reload():
         assert mod.a == 11
         assert mod.b == 20
 
-        # Now cause a `LexException`, and confirm that the good module and its
-        # contents stick around.
+        # Now cause a syntax error
         unlink(source)
 
         with open(source, "w") as f:
@@ -214,7 +207,7 @@ def test_reload():
             f.write("(setv a 11")
             f.write("(setv b (// 20 1))")
 
-        with pytest.raises(LexException):
+        with pytest.raises(PrematureEndOfInput):
             reload(mod)
 
         mod = sys.modules.get(TESTFN)
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 6336122..8aef923 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -149,7 +149,7 @@ def test_bin_hy_stdin_unlocatable_hytypeerror():
     # inside run_cmd.
     _, err = run_cmd("hy", """
         (import hy.errors)
-        (raise (hy.errors.HyTypeError '[] (+ "A" "Z")))""")
+        (raise (hy.errors.HyTypeError (+ "A" "Z") None '[] None))""")
     assert "AZ" in err
 
 
diff --git a/tests/test_lex.py b/tests/test_lex.py
index 19da88b..c759624 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -1,13 +1,15 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
+import traceback
+
+import pytest
 
 from math import isnan
 from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol,
                        HyString, HyDict, HyList, HySet, HyKeyword)
 from hy.lex import tokenize
 from hy.lex.exceptions import LexException, PrematureEndOfInput
-import pytest
 
 def peoi(): return pytest.raises(PrematureEndOfInput)
 def lexe(): return pytest.raises(LexException)
@@ -180,7 +182,23 @@ def test_lex_digit_separators():
 
 
 def test_lex_bad_attrs():
-    with lexe(): tokenize("1.foo")
+    with lexe() as execinfo:
+        tokenize("1.foo")
+
+    expected = [
+        '  File "<string>", line 1\n',
+        '    1.foo\n',
+        '    ^\n',
+        ('LexException: Cannot access attribute on anything other'
+         ' than a name (in order to get attributes of expressions,'
+         ' use `(. <expression> <attr>)` or `(.<attr> <expression>)`)\n')
+    ]
+    output = traceback.format_exception_only(execinfo.type, execinfo.value)
+
+    assert output[:-1:1] == expected[:-1:1]
+    # Python 2.7 doesn't give the full exception name, so we compensate.
+    assert output[-1].endswith(expected[-1])
+
     with lexe(): tokenize("0.foo")
     with lexe(): tokenize("1.5.foo")
     with lexe(): tokenize("1e3.foo")

From e468d5f081af576a3321a0d4a5007ed5422095c0 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sun, 28 Oct 2018 21:42:18 -0500
Subject: [PATCH 101/223] Refactor REPL error handling and filter Hy internal
 trace output

These changes make the Hy REPL more closely follow `code.InteractiveConsole`'s
class interface and provide minimally intrusive traceback print-out filtering
via a context manager that temporarily alters `sys.excepthook`.  In other words,
exception messages from the REPL will no longer show Hy internal
code (e.g. importer, compiler and parsing functions).

The boolean variable `hy.errors._hy_filter_internal_errors` dynamically
enables/disables trace filtering, and the env variable
`HY_FILTER_INTERNAL_ERRORS` can be used as the initial value.
---
 docs/language/cli.rst |   6 --
 hy/__init__.py        |  10 ++++
 hy/cmdline.py         | 125 +++++++++++++++++++++---------------------
 hy/compiler.py        |   2 +-
 hy/errors.py          |  82 ++++++++++++++++++++++++++-
 tests/test_lex.py     |  80 ++++++++++++++++++++++-----
 6 files changed, 218 insertions(+), 87 deletions(-)

diff --git a/docs/language/cli.rst b/docs/language/cli.rst
index 2c8a1f7..e59640d 100644
--- a/docs/language/cli.rst
+++ b/docs/language/cli.rst
@@ -48,12 +48,6 @@ Command Line Options
    `--spy` only works on REPL mode.
    .. versionadded:: 0.9.11
 
-.. cmdoption:: --show-tracebacks
-
-   Print extended tracebacks for Hy exceptions.
-
-   .. versionadded:: 0.9.12
-
 .. cmdoption:: --repl-output-fn
 
    Format REPL output using specific function (e.g., hy.contrib.hy-repr.hy-repr)
diff --git a/hy/__init__.py b/hy/__init__.py
index f188b64..eb1d91c 100644
--- a/hy/__init__.py
+++ b/hy/__init__.py
@@ -5,6 +5,16 @@ except ImportError:
     __version__ = 'unknown'
 
 
+def _initialize_env_var(env_var, default_val):
+    import os, distutils.util
+    try:
+        res = bool(distutils.util.strtobool(
+            os.environ.get(env_var, str(default_val))))
+    except ValueError as e:
+        res = default_val
+    return res
+
+
 from hy.models import HyExpression, HyInteger, HyKeyword, HyComplex, HyString, HyBytes, HySymbol, HyFloat, HyDict, HyList, HySet  # NOQA
 
 
diff --git a/hy/cmdline.py b/hy/cmdline.py
index 70d7b72..f7cca52 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -21,7 +21,7 @@ import hy
 from hy.lex import hy_parse, mangle
 from hy.lex.exceptions import PrematureEndOfInput
 from hy.compiler import HyASTCompiler, hy_compile, hy_eval
-from hy.errors import HyTypeError, HyLanguageError, HySyntaxError
+from hy.errors import HySyntaxError, filtered_hy_exceptions
 from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
@@ -90,47 +90,58 @@ class HyREPL(code.InteractiveConsole, object):
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
 
-    def runsource(self, source, filename='<input>', symbol='single'):
-        global SIMPLE_TRACEBACKS
+    def ast_callback(self, main_ast, expr_ast):
+        if self.spy:
+            # Mush the two AST chunks into a single module for
+            # conversion into Python.
+            new_ast = ast.Module(main_ast.body +
+                                 [ast.Expr(expr_ast.body)])
+            print(astor.to_source(new_ast))
 
-        def error_handler(e, use_simple_traceback=False):
-            self.locals[mangle("*e")] = e
-            if use_simple_traceback:
-                print(e, file=sys.stderr)
-            else:
-                self.showtraceback()
+    def _error_wrap(self, error_fn, *args, **kwargs):
+        sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+
+        # Sadly, this method in Python 2.7 ignores an overridden
+        # `sys.excepthook`.
+        if sys.excepthook is sys.__excepthook__:
+            error_fn(*args, **kwargs)
+        else:
+            sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback)
+
+        self.locals[mangle("*e")] = sys.last_value
+
+    def showsyntaxerror(self, filename=None):
+        if filename is None:
+            filename = self.filename
+
+        self._error_wrap(super(HyREPL, self).showsyntaxerror,
+                         filename=filename)
+
+    def showtraceback(self):
+        self._error_wrap(super(HyREPL, self).showtraceback)
+
+    def runsource(self, source, filename='<input>', symbol='single'):
 
         try:
             do = hy_parse(source, filename=filename)
         except PrematureEndOfInput:
             return True
         except HySyntaxError as e:
-            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
+            self.showsyntaxerror(filename=filename)
             return False
 
         try:
-            def ast_callback(main_ast, expr_ast):
-                if self.spy:
-                    # Mush the two AST chunks into a single module for
-                    # conversion into Python.
-                    new_ast = ast.Module(main_ast.body +
-                                         [ast.Expr(expr_ast.body)])
-                    print(astor.to_source(new_ast))
-
-            value = hy_eval(do, self.locals, self.module,
-                            ast_callback=ast_callback,
-                            compiler=self.hy_compiler,
-                            filename=filename,
+            # Our compiler doesn't correspond to a real, fixed source file, so
+            # we need to [re]set these.
+            self.hy_compiler.filename = filename
+            self.hy_compiler.source = source
+            value = hy_eval(do, self.locals, self.module, self.ast_callback,
+                            compiler=self.hy_compiler, filename=filename,
                             source=source)
-
-        except HyTypeError as e:
-            if e.source is None:
-                e.source = source
-                e.filename = filename
-            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
-            return False
+        except SystemExit:
+            raise
         except Exception as e:
-            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
+            self.showtraceback()
             return False
 
         if value is not None:
@@ -142,10 +153,12 @@ class HyREPL(code.InteractiveConsole, object):
             # Print the value.
             try:
                 output = self.output_fn(value)
-            except Exception as e:
-                error_handler(e)
+            except Exception:
+                self.showtraceback()
                 return False
+
             print(output)
+
         return False
 
 
@@ -201,25 +214,12 @@ def ideas_macro(ETname):
 """)])
 
 
-SIMPLE_TRACEBACKS = True
-
-
-def pretty_error(func, *args, **kw):
-    try:
-        return func(*args, **kw)
-    except HyLanguageError as e:
-        if SIMPLE_TRACEBACKS:
-            print(e, file=sys.stderr)
-            sys.exit(1)
-        raise
-
-
 def run_command(source, filename=None):
     tree = hy_parse(source, filename=filename)
     __main__ = importlib.import_module('__main__')
     require("hy.cmdline", __main__, assignments="ALL")
-    pretty_error(hy_eval, tree, None, __main__, filename=filename,
-                 source=source)
+    with filtered_hy_exceptions():
+        hy_eval(tree, None, __main__, filename=filename, source=source)
     return 0
 
 
@@ -232,9 +232,7 @@ def run_repl(hr=None, **kwargs):
         hr = HyREPL(**kwargs)
 
     namespace = hr.locals
-
-    with completion(Completer(namespace)):
-
+    with filtered_hy_exceptions(), completion(Completer(namespace)):
         hr.interact("{appname} {version} using "
                     "{py}({build}) {pyversion} on {os}".format(
                         appname=hy.__appname__,
@@ -263,9 +261,10 @@ def run_icommand(source, **kwargs):
     else:
         filename = '<input>'
 
-    hr = HyREPL(**kwargs)
-    hr.runsource(source, filename=filename, symbol='single')
-    return run_repl(hr)
+    with filtered_hy_exceptions():
+        hr = HyREPL(**kwargs)
+        hr.runsource(source, filename=filename, symbol='single')
+        return run_repl(hr)
 
 
 USAGE = "%(prog)s [-h | -i cmd | -c cmd | -m module | file | -] [arg] ..."
@@ -301,9 +300,6 @@ def cmdline_handler(scriptname, argv):
                              "(e.g., hy.contrib.hy-repr.hy-repr)")
     parser.add_argument("-v", "--version", action="version", version=VERSION)
 
-    parser.add_argument("--show-tracebacks", action="store_true",
-                        help="show complete tracebacks for Hy exceptions")
-
     # this will contain the script/program name and any arguments for it.
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
@@ -328,10 +324,6 @@ def cmdline_handler(scriptname, argv):
 
     options = parser.parse_args(argv[1:])
 
-    if options.show_tracebacks:
-        global SIMPLE_TRACEBACKS
-        SIMPLE_TRACEBACKS = False
-
     if options.E:
         # User did "hy -E ..."
         _remove_python_envs()
@@ -372,7 +364,8 @@ def cmdline_handler(scriptname, argv):
 
             try:
                 sys.argv = options.args
-                runhy.run_path(filename, run_name='__main__')
+                with filtered_hy_exceptions():
+                    runhy.run_path(filename, run_name='__main__')
                 return 0
             except FileNotFoundError as e:
                 print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
@@ -448,9 +441,11 @@ def hy2py_main():
 
     if options.FILE is None or options.FILE == '-':
         source = sys.stdin.read()
-        hst = pretty_error(hy_parse, source, filename='<stdin>')
+        with filtered_hy_exceptions():
+            hst = hy_parse(source, filename='<stdin>')
     else:
-        with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
+        with filtered_hy_exceptions(), \
+             io.open(options.FILE, 'r', encoding='utf-8') as source_file:
             source = source_file.read()
             hst = hy_parse(source, filename=options.FILE)
 
@@ -468,7 +463,9 @@ def hy2py_main():
         print()
         print()
 
-    _ast = pretty_error(hy_compile, hst, '__main__')
+    with filtered_hy_exceptions():
+        _ast = hy_compile(hst, '__main__')
+
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))
diff --git a/hy/compiler.py b/hy/compiler.py
index 5377c33..a02e71a 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1834,7 +1834,7 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False):
 
 
 def hy_eval(hytree, locals=None, module=None, ast_callback=None,
-            compiler=None, filename='<string>', source=None):
+            compiler=None, filename=None, source=None):
     """Evaluates a quoted expression and returns the value.
 
     If you're evaluating hand-crafted AST trees, make sure the line numbers
diff --git a/hy/errors.py b/hy/errors.py
index e8e9f98..4ed56e5 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -2,13 +2,20 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
-
+import os
+import sys
 import traceback
+import pkgutil
 
 from functools import reduce
+from contextlib import contextmanager
+from hy import _initialize_env_var
 
 from clint.textui import colored
 
+_hy_filter_internal_errors = _initialize_env_var('HY_FILTER_INTERNAL_ERRORS',
+                                                 True)
+
 
 class HyError(Exception):
     def __init__(self, message, *args):
@@ -193,7 +200,8 @@ class HySyntaxError(HyLanguageError, SyntaxError):
 
     def __str__(self):
 
-        output = traceback.format_exception_only(SyntaxError, self)
+        output = traceback.format_exception_only(SyntaxError,
+                                                 SyntaxError(*self.args))
 
         output[-1] = colored.yellow(output[-1])
         if len(self.source) > 0:
@@ -206,3 +214,73 @@ class HySyntaxError(HyLanguageError, SyntaxError):
 
         # Avoid "...expected str instance, ColoredString found"
         return reduce(lambda x, y: x + y, output)
+
+
+def _get_module_info(module):
+    compiler_loader = pkgutil.get_loader(module)
+    is_pkg = compiler_loader.is_package(module)
+    filename = compiler_loader.get_filename()
+    if is_pkg:
+        # Use package directory
+        return os.path.dirname(filename)
+    else:
+        # Normalize filename endings, because tracebacks will use `pyc` when
+        # the loader says `py`.
+        return filename.replace('.pyc', '.py')
+
+
+_tb_hidden_modules = {_get_module_info(m)
+                      for m in ['hy.compiler', 'hy.lex',
+                                'hy.cmdline', 'hy.lex.parser',
+                                'hy.importer', 'hy._compat',
+                                'hy.macros', 'hy.models',
+                                'rply']}
+
+
+def hy_exc_handler(exc_type, exc_value, exc_traceback):
+    """Produce exceptions print-outs with all frames originating from the
+    modules in `_tb_hidden_modules` filtered out.
+
+    The frames are actually filtered by each module's filename and only when a
+    subclass of `HyLanguageError` is emitted.
+
+    This does not remove the frames from the actual tracebacks, so debugging
+    will show everything.
+    """
+    try:
+        # frame = (filename, line number, function name*, text)
+        new_tb = [frame for frame in traceback.extract_tb(exc_traceback)
+                  if not (frame[0].replace('.pyc', '.py') in _tb_hidden_modules or
+                          os.path.dirname(frame[0]) in _tb_hidden_modules)]
+
+        lines = traceback.format_list(new_tb)
+
+        if lines:
+            lines.insert(0, "Traceback (most recent call last):\n")
+
+        lines.extend(traceback.format_exception_only(exc_type, exc_value))
+        output = ''.join(lines)
+
+        sys.stderr.write(output)
+        sys.stderr.flush()
+    except Exception:
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
+
+
+@contextmanager
+def filtered_hy_exceptions():
+    """Temporarily apply a `sys.excepthook` that filters Hy internal frames
+    from tracebacks.
+
+    Filtering can be controlled by the variable
+    `hy.errors._hy_filter_internal_errors` and environment variable
+    `HY_FILTER_INTERNAL_ERRORS`.
+    """
+    global _hy_filter_internal_errors
+    if _hy_filter_internal_errors:
+        current_hook = sys.excepthook
+        sys.excepthook = hy_exc_handler
+        yield
+        sys.excepthook = current_hook
+    else:
+        yield
diff --git a/tests/test_lex.py b/tests/test_lex.py
index c759624..b0a03dc 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -1,6 +1,7 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
+import sys
 import traceback
 
 import pytest
@@ -10,11 +11,36 @@ from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol,
                        HyString, HyDict, HyList, HySet, HyKeyword)
 from hy.lex import tokenize
 from hy.lex.exceptions import LexException, PrematureEndOfInput
+from hy.errors import hy_exc_handler
 
 def peoi(): return pytest.raises(PrematureEndOfInput)
 def lexe(): return pytest.raises(LexException)
 
 
+def check_ex(execinfo, expected):
+    output = traceback.format_exception_only(execinfo.type, execinfo.value)
+    assert output[:-1] == expected[:-1]
+    # Python 2.7 doesn't give the full exception name, so we compensate.
+    assert output[-1].endswith(expected[-1])
+
+
+def check_trace_output(capsys, execinfo, expected):
+   sys.__excepthook__(execinfo.type, execinfo.value, execinfo.tb)
+   captured_wo_filtering = capsys.readouterr()[-1].strip('\n')
+
+   hy_exc_handler(execinfo.type, execinfo.value, execinfo.tb)
+   captured_w_filtering = capsys.readouterr()[-1].strip('\n')
+
+   output = captured_w_filtering.split('\n')
+
+   # Make sure the filtered frames aren't the same as the unfiltered ones.
+   assert output[:-1] != captured_wo_filtering.split('\n')[:-1]
+   # Remove the origin frame lines.
+   assert output[3:-1] == expected[:-1]
+   # Python 2.7 doesn't give the full exception name, so we compensate.
+   assert output[-1].endswith(expected[-1])
+
+
 def test_lex_exception():
     """ Ensure tokenize throws a fit on a partial input """
     with peoi(): tokenize("(foo")
@@ -32,8 +58,13 @@ def test_unbalanced_exception():
 def test_lex_single_quote_err():
     "Ensure tokenizing \"' \" throws a LexException that can be stringified"
     # https://github.com/hylang/hy/issues/1252
-    with lexe() as e: tokenize("' ")
-    assert "Could not identify the next token" in str(e.value)
+    with lexe() as execinfo:
+        tokenize("' ")
+    check_ex(execinfo, [
+        '  File "<string>", line -1\n',
+        "    '\n",
+        '    ^\n',
+        'LexException: Could not identify the next token.\n'])
 
 
 def test_lex_expression_symbols():
@@ -76,7 +107,11 @@ def test_lex_strings_exception():
     """ Make sure tokenize throws when codec can't decode some bytes"""
     with lexe() as execinfo:
         tokenize('\"\\x8\"')
-    assert "Can't convert \"\\x8\" to a HyString" in str(execinfo.value)
+    check_ex(execinfo, [
+        '  File "<string>", line 1\n',
+        '    "\\x8"\n',
+        '    ^\n',
+        'LexException: Can\'t convert "\\x8" to a HyString\n'])
 
 
 def test_lex_bracket_strings():
@@ -184,20 +219,13 @@ def test_lex_digit_separators():
 def test_lex_bad_attrs():
     with lexe() as execinfo:
         tokenize("1.foo")
-
-    expected = [
+    check_ex(execinfo, [
         '  File "<string>", line 1\n',
         '    1.foo\n',
         '    ^\n',
-        ('LexException: Cannot access attribute on anything other'
-         ' than a name (in order to get attributes of expressions,'
-         ' use `(. <expression> <attr>)` or `(.<attr> <expression>)`)\n')
-    ]
-    output = traceback.format_exception_only(execinfo.type, execinfo.value)
-
-    assert output[:-1:1] == expected[:-1:1]
-    # Python 2.7 doesn't give the full exception name, so we compensate.
-    assert output[-1].endswith(expected[-1])
+        'LexException: Cannot access attribute on anything other'
+            ' than a name (in order to get attributes of expressions,'
+            ' use `(. <expression> <attr>)` or `(.<attr> <expression>)`)\n'])
 
     with lexe(): tokenize("0.foo")
     with lexe(): tokenize("1.5.foo")
@@ -437,3 +465,27 @@ def test_discard():
     assert tokenize("a '#_b c") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("c")])]
     assert tokenize("a '#_b #_c d") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("d")])]
     assert tokenize("a '#_ #_b c d") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("d")])]
+
+
+def test_lex_exception_filtering(capsys):
+    """Confirm that the exception filtering works for lexer errors."""
+
+    # First, test for PrematureEndOfInput
+    with peoi() as execinfo:
+        tokenize(" \n (foo")
+    check_trace_output(capsys, execinfo, [
+        '  File "<string>", line 2',
+        '    (foo',
+        '       ^',
+        'PrematureEndOfInput: Premature end of input'])
+
+    # Now, for a generic LexException
+    with lexe() as execinfo:
+        tokenize("  \n\n  1.foo   ")
+    check_trace_output(capsys, execinfo, [
+        '  File "<string>", line 3',
+        '    1.foo',
+        '    ^',
+        'LexException: Cannot access attribute on anything other'
+            ' than a name (in order to get attributes of expressions,'
+            ' use `(. <expression> <attr>)` or `(.<attr> <expression>)`)'])

From cadfa4152bca314e2e614baa19e2ccbf07ade00e Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sun, 28 Oct 2018 22:02:08 -0500
Subject: [PATCH 102/223] Make colored output configurable

Colored exception output is now disabled by default and configurable through
`hy.errors._hy_colored_errors` and the environment variable
`HY_COLORED_ERRORS`.

Likewise, Hy model/AST color printing is now configurable and disabled by
default.  The corresponding variables are `hy.models._hy_colored_ast_objects`
and `HY_COLORED_AST_OBJECTS`.

Closes hylang/hy#1429, closes hylang/hy#1510.
---
 hy/errors.py | 49 ++++++++++++++++++++++++++++++-------------------
 hy/models.py | 12 ++++++++----
 2 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/hy/errors.py b/hy/errors.py
index 4ed56e5..9ca823e 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -15,6 +15,7 @@ from clint.textui import colored
 
 _hy_filter_internal_errors = _initialize_env_var('HY_FILTER_INTERNAL_ERRORS',
                                                  True)
+_hy_colored_errors = _initialize_env_var('HY_COLORED_ERRORS', False)
 
 
 class HyError(Exception):
@@ -73,9 +74,16 @@ class HyTypeError(HyLanguageError, TypeError):
                                           source)
 
     def __str__(self):
+        global _hy_colored_errors
 
         result = ""
 
+        if _hy_colored_errors:
+            from clint.textui import colored
+            red, green, yellow = colored.red, colored.green, colored.yellow
+        else:
+            red = green = yellow = lambda x: x
+
         if all(getattr(self.expression, x, None) is not None
                for x in ("start_line", "start_column", "end_column")):
 
@@ -97,28 +105,28 @@ class HyTypeError(HyLanguageError, TypeError):
                                                                start)
 
             if len(source) == 1:
-                result += '  %s\n' % colored.red(source[0])
+                result += '  %s\n' % red(source[0])
                 result += '  %s%s\n' % (' '*(start-1),
-                                        colored.green('^' + '-'*(length-1) + '^'))
+                                        green('^' + '-'*(length-1) + '^'))
             if len(source) > 1:
-                result += '  %s\n' % colored.red(source[0])
+                result += '  %s\n' % red(source[0])
                 result += '  %s%s\n' % (' '*(start-1),
-                                        colored.green('^' + '-'*length))
+                                        green('^' + '-'*length))
                 if len(source) > 2:  # write the middle lines
                     for line in source[1:-1]:
-                        result += '  %s\n' % colored.red("".join(line))
-                        result += '  %s\n' % colored.green("-"*len(line))
+                        result += '  %s\n' % red("".join(line))
+                        result += '  %s\n' % green("-"*len(line))
 
                 # write the last line
-                result += '  %s\n' % colored.red("".join(source[-1]))
-                result += '  %s\n' % colored.green('-'*(end-1) + '^')
+                result += '  %s\n' % red("".join(source[-1]))
+                result += '  %s\n' % green('-'*(end-1) + '^')
 
         else:
             result += '  File "%s", unknown location\n' % self.filename
 
-        result += colored.yellow("%s: %s\n\n" %
-                                 (self.__class__.__name__,
-                                  self.message))
+        result += yellow("%s: %s\n\n" %
+                         (self.__class__.__name__,
+                          self.message))
 
         return result
 
@@ -199,18 +207,21 @@ class HySyntaxError(HyLanguageError, SyntaxError):
         return HySyntaxError(message, filename, lineno, colno, source)
 
     def __str__(self):
+        global _hy_colored_errors
 
         output = traceback.format_exception_only(SyntaxError,
                                                  SyntaxError(*self.args))
 
-        output[-1] = colored.yellow(output[-1])
-        if len(self.source) > 0:
-            output[-2] = colored.green(output[-2])
-            for line in output[::-2]:
-                if line.strip().startswith(
-                        'File "{}", line'.format(self.filename)):
-                    break
-            output[-3] = colored.red(output[-3])
+        if _hy_colored_errors:
+            from hy.errors import colored
+            output[-1] = colored.yellow(output[-1])
+            if len(self.source) > 0:
+                output[-2] = colored.green(output[-2])
+                for line in output[::-2]:
+                    if line.strip().startswith(
+                            'File "{}", line'.format(self.filename)):
+                        break
+                output[-3] = colored.red(output[-3])
 
         # Avoid "...expected str instance, ColoredString found"
         return reduce(lambda x, y: x + y, output)
diff --git a/hy/models.py b/hy/models.py
index cf02dab..134e322 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -1,16 +1,17 @@
 # Copyright 2019 the authors.
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
-
 from __future__ import unicode_literals
+
 from contextlib import contextmanager
 from math import isnan, isinf
+from hy import _initialize_env_var
 from hy._compat import PY3, str_type, bytes_type, long_type, string_types
 from fractions import Fraction
 from clint.textui import colored
 
-
 PRETTY = True
+_hy_colored_ast_objects = _initialize_env_var('HY_COLORED_AST_OBJECTS', False)
 
 
 @contextmanager
@@ -271,8 +272,9 @@ class HySequence(HyObject, list):
         return str(self) if PRETTY else super(HySequence, self).__repr__()
 
     def __str__(self):
+        global _hy_colored_ast_objects
         with pretty():
-            c = self.color
+            c = self.color if _hy_colored_ast_objects else str
             if self:
                 return ("{}{}\n  {}{}").format(
                     c(self.__class__.__name__),
@@ -298,10 +300,12 @@ class HyDict(HySequence):
     """
     HyDict (just a representation of a dict)
     """
+    color = staticmethod(colored.green)
 
     def __str__(self):
+        global _hy_colored_ast_objects
         with pretty():
-            g = colored.green
+            g = self.color if _hy_colored_ast_objects else str
             if self:
                 pairs = []
                 for k, v in zip(self[::2],self[1::2]):

From fb6feaf08298d67045583e4dcae090f1e80b2b94 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Fri, 12 Oct 2018 23:25:43 -0500
Subject: [PATCH 103/223] Improve correspondence with Python errors and console
 behavior

Compiler and command-line error messages now reflect their Python counterparts.
E.g. where Python emits a `SyntaxError`, so does Hy; same for `TypeError`s.
Multiple tests have been added that check the format and type of raised
exceptions over varying command-line invocations (e.g. interactive and not).

A new exception type for `require` errors was added so that they can be treated
like normal run-time errors and not compiler errors.

The Hy REPL has been further refactored to better match the class-structured
API.  Now, different error types are handled separately and leverage more base
class-provided functionality.

Closes hylang/hy#1486.
---
 hy/_compat.py                        |  24 ++
 hy/cmdline.py                        | 173 +++++++++----
 hy/compiler.py                       |  13 +-
 hy/core/bootstrap.hy                 |  10 +-
 hy/errors.py                         | 369 ++++++++++++++-------------
 hy/lex/__init__.py                   |  26 +-
 hy/lex/exceptions.py                 |  35 +--
 hy/lex/parser.py                     |  10 -
 hy/macros.py                         |  79 +++---
 tests/compilers/test_ast.py          |  56 ++--
 tests/macros/test_macro_processor.py |   3 +-
 tests/native_tests/core.hy           |   2 +-
 tests/native_tests/language.hy       |  30 +--
 tests/native_tests/native_macros.hy  |  31 ++-
 tests/native_tests/operators.hy      |   2 +-
 tests/test_bin.py                    | 151 ++++++++++-
 tests/test_lex.py                    |   4 +-
 17 files changed, 630 insertions(+), 388 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 55180ca..2711445 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -40,6 +40,10 @@ if PY3:
         finally:
             traceback = None
 
+    code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize',
+                     'flags', 'code', 'consts', 'names', 'varnames',
+                     'filename', 'name', 'firstlineno', 'lnotab', 'freevars',
+                     'cellvars']
 else:
     def raise_from(value, from_value=None):
         raise value
@@ -52,10 +56,30 @@ else:
             traceback = None
     ''')
 
+    code_obj_args = ['argcount', 'nlocals', 'stacksize', 'flags', 'code',
+                     'consts', 'names', 'varnames', 'filename', 'name',
+                     'firstlineno', 'lnotab', 'freevars', 'cellvars']
+
 raise_code = compile(raise_src, __file__, 'exec')
 exec(raise_code)
 
 
+def rename_function(func, new_name):
+    """Creates a copy of a function and [re]sets the name at the code-object
+    level.
+    """
+    c = func.__code__
+    new_code = type(c)(*[getattr(c, 'co_{}'.format(a))
+                         if a != 'name' else str(new_name)
+                         for a in code_obj_args])
+
+    _fn = type(func)(new_code, func.__globals__, str(new_name),
+                     func.__defaults__, func.__closure__)
+    _fn.__dict__.update(func.__dict__)
+
+    return _fn
+
+
 def isidentifier(x):
     if x in ('True', 'False', 'None', 'print'):
         # `print` is special-cased here because Python 2's
diff --git a/hy/cmdline.py b/hy/cmdline.py
index f7cca52..a9f3af3 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -12,6 +12,7 @@ import os
 import io
 import importlib
 import py_compile
+import traceback
 import runpy
 import types
 
@@ -20,8 +21,9 @@ import astor.code_gen
 import hy
 from hy.lex import hy_parse, mangle
 from hy.lex.exceptions import PrematureEndOfInput
-from hy.compiler import HyASTCompiler, hy_compile, hy_eval
-from hy.errors import HySyntaxError, filtered_hy_exceptions
+from hy.compiler import HyASTCompiler, hy_eval, hy_compile, ast_compile
+from hy.errors import (HyLanguageError, HyRequireError, HyMacroExpansionError,
+                       filtered_hy_exceptions, hy_exc_handler)
 from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
@@ -50,29 +52,70 @@ builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
 
+class HyCommandCompiler(object):
+    def __init__(self, module, ast_callback=None, hy_compiler=None):
+        self.module = module
+        self.ast_callback = ast_callback
+        self.hy_compiler = hy_compiler
+
+    def __call__(self, source, filename="<input>", symbol="single"):
+        try:
+            hy_ast = hy_parse(source, filename=filename)
+            root_ast = ast.Interactive if symbol == 'single' else ast.Module
+
+            # Our compiler doesn't correspond to a real, fixed source file, so
+            # we need to [re]set these.
+            self.hy_compiler.filename = filename
+            self.hy_compiler.source = source
+            exec_ast, eval_ast = hy_compile(hy_ast, self.module, root=root_ast,
+                                            get_expr=True,
+                                            compiler=self.hy_compiler,
+                                            filename=filename, source=source)
+
+            if self.ast_callback:
+                self.ast_callback(exec_ast, eval_ast)
+
+            exec_code = ast_compile(exec_ast, filename, symbol)
+            eval_code = ast_compile(eval_ast, filename, 'eval')
+
+            return exec_code, eval_code
+        except PrematureEndOfInput:
+            # Save these so that we can reraise/display when an incomplete
+            # interactive command is given at the prompt.
+            sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+            return None
+
+
 class HyREPL(code.InteractiveConsole, object):
     def __init__(self, spy=False, output_fn=None, locals=None,
-                 filename="<input>"):
-
-        super(HyREPL, self).__init__(locals=locals,
-                                     filename=filename)
+                 filename="<stdin>"):
 
         # Create a proper module for this REPL so that we can obtain it easily
         # (e.g. using `importlib.import_module`).
-        # Also, make sure it's properly introduced to `sys.modules` and
-        # consistently use its namespace as `locals` from here on.
+        # We let `InteractiveConsole` initialize `self.locals` when it's
+        # `None`.
+        super(HyREPL, self).__init__(locals=locals,
+                                     filename=filename)
+
         module_name = self.locals.get('__name__', '__console__')
+        # Make sure our newly created module is properly introduced to
+        # `sys.modules`, and consistently use its namespace as `self.locals`
+        # from here on.
         self.module = sys.modules.setdefault(module_name,
                                              types.ModuleType(module_name))
         self.module.__dict__.update(self.locals)
         self.locals = self.module.__dict__
 
         # Load cmdline-specific macros.
-        require('hy.cmdline', module_name, assignments='ALL')
+        require('hy.cmdline', self.module, assignments='ALL')
 
         self.hy_compiler = HyASTCompiler(self.module)
 
+        self.compile = HyCommandCompiler(self.module, self.ast_callback,
+                                         self.hy_compiler)
+
         self.spy = spy
+        self.last_value = None
 
         if output_fn is None:
             self.output_fn = repr
@@ -90,13 +133,18 @@ class HyREPL(code.InteractiveConsole, object):
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
 
-    def ast_callback(self, main_ast, expr_ast):
+    def ast_callback(self, exec_ast, eval_ast):
         if self.spy:
-            # Mush the two AST chunks into a single module for
-            # conversion into Python.
-            new_ast = ast.Module(main_ast.body +
-                                 [ast.Expr(expr_ast.body)])
-            print(astor.to_source(new_ast))
+            try:
+                # Mush the two AST chunks into a single module for
+                # conversion into Python.
+                new_ast = ast.Module(exec_ast.body +
+                                     [ast.Expr(eval_ast.body)])
+                print(astor.to_source(new_ast))
+            except Exception:
+                msg = 'Exception in AST callback:\n{}\n'.format(
+                    traceback.format_exc())
+                self.write(msg)
 
     def _error_wrap(self, error_fn, *args, **kwargs):
         sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
@@ -120,46 +168,50 @@ class HyREPL(code.InteractiveConsole, object):
     def showtraceback(self):
         self._error_wrap(super(HyREPL, self).showtraceback)
 
-    def runsource(self, source, filename='<input>', symbol='single'):
-
+    def runcode(self, code):
         try:
-            do = hy_parse(source, filename=filename)
-        except PrematureEndOfInput:
-            return True
-        except HySyntaxError as e:
-            self.showsyntaxerror(filename=filename)
-            return False
-
-        try:
-            # Our compiler doesn't correspond to a real, fixed source file, so
-            # we need to [re]set these.
-            self.hy_compiler.filename = filename
-            self.hy_compiler.source = source
-            value = hy_eval(do, self.locals, self.module, self.ast_callback,
-                            compiler=self.hy_compiler, filename=filename,
-                            source=source)
+            eval(code[0], self.locals)
+            self.last_value = eval(code[1], self.locals)
+            # Don't print `None` values.
+            self.print_last_value = self.last_value is not None
         except SystemExit:
             raise
         except Exception as e:
+            # Set this to avoid a print-out of the last value on errors.
+            self.print_last_value = False
+            self.showtraceback()
+
+    def runsource(self, source, filename='<stdin>', symbol='exec'):
+        try:
+            res = super(HyREPL, self).runsource(source, filename, symbol)
+        except (HyMacroExpansionError, HyRequireError):
+            # We need to handle these exceptions ourselves, because the base
+            # method only handles `OverflowError`, `SyntaxError` and
+            # `ValueError`.
+            self.showsyntaxerror(filename)
+            return False
+        except (HyLanguageError):
+            # Our compiler will also raise `TypeError`s
             self.showtraceback()
             return False
 
-        if value is not None:
-            # Shift exisitng REPL results
-            next_result = value
+        # Shift exisitng REPL results
+        if not res:
+            next_result = self.last_value
             for sym in self._repl_results_symbols:
                 self.locals[sym], next_result = next_result, self.locals[sym]
 
             # Print the value.
-            try:
-                output = self.output_fn(value)
-            except Exception:
-                self.showtraceback()
-                return False
+            if self.print_last_value:
+                try:
+                    output = self.output_fn(self.last_value)
+                except Exception:
+                    self.showtraceback()
+                    return False
 
-            print(output)
+                print(output)
 
-        return False
+        return res
 
 
 @macro("koan")
@@ -215,9 +267,14 @@ def ideas_macro(ETname):
 
 
 def run_command(source, filename=None):
-    tree = hy_parse(source, filename=filename)
     __main__ = importlib.import_module('__main__')
     require("hy.cmdline", __main__, assignments="ALL")
+    try:
+        tree = hy_parse(source, filename=filename)
+    except HyLanguageError:
+        hy_exc_handler(*sys.exc_info())
+        return 1
+
     with filtered_hy_exceptions():
         hy_eval(tree, None, __main__, filename=filename, source=source)
     return 0
@@ -259,12 +316,18 @@ def run_icommand(source, **kwargs):
             source = f.read()
         filename = source
     else:
-        filename = '<input>'
+        filename = '<string>'
 
+    hr = HyREPL(**kwargs)
     with filtered_hy_exceptions():
-        hr = HyREPL(**kwargs)
-        hr.runsource(source, filename=filename, symbol='single')
-        return run_repl(hr)
+        res = hr.runsource(source, filename=filename)
+
+    # If the command was prematurely ended, show an error (just like Python
+    # does).
+    if res:
+        hy_exc_handler(sys.last_type, sys.last_value, sys.last_traceback)
+
+    return run_repl(hr)
 
 
 USAGE = "%(prog)s [-h | -i cmd | -c cmd | -m module | file | -] [arg] ..."
@@ -371,6 +434,9 @@ def cmdline_handler(scriptname, argv):
                 print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
                       e.filename, e.errno, e.strerror), file=sys.stderr)
                 sys.exit(e.errno)
+            except HyLanguageError:
+                hy_exc_handler(*sys.exc_info())
+                sys.exit(1)
 
     # User did NOTHING!
     return run_repl(spy=options.spy, output_fn=options.repl_output_fn)
@@ -440,14 +506,15 @@ def hy2py_main():
     options = parser.parse_args(sys.argv[1:])
 
     if options.FILE is None or options.FILE == '-':
+        filename = '<stdin>'
         source = sys.stdin.read()
-        with filtered_hy_exceptions():
-            hst = hy_parse(source, filename='<stdin>')
     else:
-        with filtered_hy_exceptions(), \
-             io.open(options.FILE, 'r', encoding='utf-8') as source_file:
+        filename = options.FILE
+        with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
             source = source_file.read()
-            hst = hy_parse(source, filename=options.FILE)
+
+    with filtered_hy_exceptions():
+        hst = hy_parse(source, filename=filename)
 
     if options.with_source:
         # need special printing on Windows in case the
@@ -464,7 +531,7 @@ def hy2py_main():
         print()
 
     with filtered_hy_exceptions():
-        _ast = hy_compile(hst, '__main__')
+        _ast = hy_compile(hst, '__main__', filename=filename, source=source)
 
     if options.with_ast:
         if PY3 and platform.system() == "Windows":
diff --git a/hy/compiler.py b/hy/compiler.py
index a02e71a..08e0c98 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -9,8 +9,8 @@ from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
 from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole,
                                notpexpr, dolike, pexpr, times, Tag, tag, unpack)
 from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
-from hy.errors import (HyCompileError, HyTypeError, HyEvalError,
-                       HyInternalError)
+from hy.errors import (HyCompileError, HyTypeError, HyLanguageError,
+                       HySyntaxError, HyEvalError, HyInternalError)
 
 from hy.lex import mangle, unmangle
 
@@ -443,15 +443,18 @@ class HyASTCompiler(object):
             # nested; so let's re-raise this exception, let's not wrap it in
             # another HyCompileError!
             raise
-        except HyTypeError as e:
-            reraise(type(e), e, None)
+        except HyLanguageError as e:
+            # These are expected errors that should be passed to the user.
+            reraise(type(e), e, sys.exc_info()[2])
         except Exception as e:
+            # These are unexpected errors that will--hopefully--never be seen
+            # by the user.
             f_exc = traceback.format_exc()
             exc_msg = "Internal Compiler Bug 😱\n⤷ {}".format(f_exc)
             reraise(HyCompileError, HyCompileError(exc_msg), sys.exc_info()[2])
 
     def _syntax_error(self, expr, message):
-        return HyTypeError(message, self.filename, expr, self.source)
+        return HySyntaxError(message, expr, self.filename, self.source)
 
     def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
                          oldpy_unpack=False):
diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy
index 0c98f60..0d3d737 100644
--- a/hy/core/bootstrap.hy
+++ b/hy/core/bootstrap.hy
@@ -15,13 +15,13 @@
           (raise
             (hy.errors.HyTypeError
               (% "received a `%s' instead of a symbol for macro name"
-                 (. (type name) --name--))
-              --file-- macro-name None)))
+                 (. (type name) __name__))
+              None --file-- None)))
      (for [kw '[&kwonly &kwargs]]
        (if* (in kw lambda-list)
             (raise (hy.errors.HyTypeError (% "macros cannot use %s"
                                              kw)
-                                          --file-- macro-name None))))
+                                          macro-name --file-- None))))
      ;; this looks familiar...
      `(eval-and-compile
         (import hy)
@@ -46,10 +46,10 @@
       (raise (hy.errors.HyTypeError
                (% "received a `%s' instead of a symbol for tag macro name"
                   (. (type tag-name) --name--))
-               --file-- tag-name None)))
+               tag-name --file-- None)))
   (if (or (= tag-name ":")
           (= tag-name "&"))
-      (raise (NameError (% "%s can't be used as a tag macro name" tag-name))))
+      (raise (hy.errors.HyNameError (% "%s can't be used as a tag macro name" tag-name))))
   (setv tag-name (.replace (hy.models.HyString tag-name)
                            tag-name))
   `(eval-and-compile
diff --git a/hy/errors.py b/hy/errors.py
index 9ca823e..0579e96 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -3,6 +3,7 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 import os
+import re
 import sys
 import traceback
 import pkgutil
@@ -19,9 +20,7 @@ _hy_colored_errors = _initialize_env_var('HY_COLORED_ERRORS', False)
 
 
 class HyError(Exception):
-    def __init__(self, message, *args):
-        self.message = message
-        super(HyError, self).__init__(message, *args)
+    pass
 
 
 class HyInternalError(HyError):
@@ -31,9 +30,6 @@ class HyInternalError(HyError):
     hopefully, never be seen by users!
     """
 
-    def __init__(self, message, *args):
-        super(HyInternalError, self).__init__(message, *args)
-
 
 class HyLanguageError(HyError):
     """Errors caused by invalid use of the Hy language.
@@ -41,8 +37,127 @@ class HyLanguageError(HyError):
     This, and any errors inheriting from this, are user-facing.
     """
 
-    def __init__(self, message, *args):
-        super(HyLanguageError, self).__init__(message, *args)
+    def __init__(self, message, expression=None, filename=None, source=None,
+                 lineno=1, colno=1):
+        """
+        Parameters
+        ----------
+        message: str
+            The message to display for this error.
+        expression: HyObject, optional
+            The Hy expression generating this error.
+        filename: str, optional
+            The filename for the source code generating this error.
+            Expression-provided information will take precedence of this value.
+        source: str, optional
+            The actual source code generating this error.  Expression-provided
+            information will take precedence of this value.
+        lineno: int, optional
+            The line number of the error.  Expression-provided information will
+            take precedence of this value.
+        colno: int, optional
+            The column number of the error.  Expression-provided information
+            will take precedence of this value.
+        """
+        self.msg = message
+        self.compute_lineinfo(expression, filename, source, lineno, colno)
+
+        if isinstance(self, SyntaxError):
+            syntax_error_args = (self.filename, self.lineno, self.offset,
+                                 self.text)
+            super(HyLanguageError, self).__init__(message, syntax_error_args)
+        else:
+            super(HyLanguageError, self).__init__(message)
+
+    def compute_lineinfo(self, expression, filename, source, lineno, colno):
+
+        # NOTE: We use `SyntaxError`'s field names (i.e. `text`, `offset`,
+        # `msg`) for compatibility and print-outs.
+        self.text = getattr(expression, 'source', source)
+        self.filename = getattr(expression, 'filename', filename)
+
+        if self.text:
+            lines = self.text.splitlines()
+
+            self.lineno = getattr(expression, 'start_line', lineno)
+            self.offset = getattr(expression, 'start_column', colno)
+            end_column = getattr(expression, 'end_column',
+                                 len(lines[self.lineno-1]))
+            end_line = getattr(expression, 'end_line', self.lineno)
+
+            # Trim the source down to the essentials.
+            self.text = '\n'.join(lines[self.lineno-1:end_line])
+
+            if end_column:
+                if self.lineno == end_line:
+                    self.arrow_offset = end_column
+                else:
+                    self.arrow_offset = len(self.text[0])
+
+                self.arrow_offset -= self.offset
+            else:
+                self.arrow_offset = None
+        else:
+            # We could attempt to extract the source given a filename, but we
+            # don't.
+            self.lineno = lineno
+            self.offset = colno
+            self.arrow_offset = None
+
+    def __str__(self):
+        """Provide an exception message that includes SyntaxError-like source
+        line information when available.
+        """
+        global _hy_colored_errors
+
+        # Syntax errors are special and annotate the traceback (instead of what
+        # we would do in the message that follows the traceback).
+        if isinstance(self, SyntaxError):
+            return super(HyLanguageError, self).__str__()
+
+        # When there isn't extra source information, use the normal message.
+        if not isinstance(self, SyntaxError) and not self.text:
+            return super(HyLanguageError, self).__str__()
+
+        # Re-purpose Python's builtin syntax error formatting.
+        output = traceback.format_exception_only(
+            SyntaxError,
+            SyntaxError(self.msg, (self.filename, self.lineno, self.offset,
+                                   self.text)))
+
+        arrow_idx, _ = next(((i, x) for i, x in enumerate(output)
+                             if x.strip() == '^'),
+                            (None, None))
+        if arrow_idx:
+            msg_idx = arrow_idx + 1
+        else:
+            msg_idx, _ = next((i, x) for i, x in enumerate(output)
+                              if x.startswith('SyntaxError: '))
+
+        # Get rid of erroneous error-type label.
+        output[msg_idx] = re.sub('^SyntaxError: ', '', output[msg_idx])
+
+        # Extend the text arrow, when given enough source info.
+        if arrow_idx and self.arrow_offset:
+            output[arrow_idx] = '{}{}^\n'.format(output[arrow_idx].rstrip('\n'),
+                                                 '-' * (self.arrow_offset - 1))
+
+        if _hy_colored_errors:
+            from clint.textui import colored
+            output[msg_idx:] = [colored.yellow(o) for o in output[msg_idx:]]
+            if arrow_idx:
+                output[arrow_idx] = colored.green(output[arrow_idx])
+            for idx, line in enumerate(output[::msg_idx]):
+                if line.strip().startswith(
+                        'File "{}", line'.format(self.filename)):
+                    output[idx] = colored.red(line)
+
+        # This resulting string will come after a "<class-name>:" prompt, so
+        # put it down a line.
+        output.insert(0, '\n')
+
+        # Avoid "...expected str instance, ColoredString found"
+        return reduce(lambda x, y: x + y, output)
 
 
 class HyCompileError(HyInternalError):
@@ -50,88 +165,21 @@ class HyCompileError(HyInternalError):
 
 
 class HyTypeError(HyLanguageError, TypeError):
-    """TypeErrors occurring during the normal use of Hy."""
-
-    def __init__(self, message, filename=None, expression=None, source=None):
-        """
-        Parameters
-        ----------
-        message: str
-            The message to display for this error.
-        filename: str, optional
-            The filename for the source code generating this error.
-        expression: HyObject, optional
-            The Hy expression generating this error.
-        source: str, optional
-            The actual source code generating this error.
-        """
-        self.message = message
-        self.filename = filename
-        self.expression = expression
-        self.source = source
-
-        super(HyTypeError, self).__init__(message, filename, expression,
-                                          source)
-
-    def __str__(self):
-        global _hy_colored_errors
-
-        result = ""
-
-        if _hy_colored_errors:
-            from clint.textui import colored
-            red, green, yellow = colored.red, colored.green, colored.yellow
-        else:
-            red = green = yellow = lambda x: x
-
-        if all(getattr(self.expression, x, None) is not None
-               for x in ("start_line", "start_column", "end_column")):
-
-            line = self.expression.start_line
-            start = self.expression.start_column
-            end = self.expression.end_column
-
-            source = []
-            if self.source is not None:
-                source = self.source.split("\n")[line-1:self.expression.end_line]
-
-                if line == self.expression.end_line:
-                    length = end - start
-                else:
-                    length = len(source[0]) - start
-
-            result += '  File "%s", line %d, column %d\n\n' % (self.filename,
-                                                               line,
-                                                               start)
-
-            if len(source) == 1:
-                result += '  %s\n' % red(source[0])
-                result += '  %s%s\n' % (' '*(start-1),
-                                        green('^' + '-'*(length-1) + '^'))
-            if len(source) > 1:
-                result += '  %s\n' % red(source[0])
-                result += '  %s%s\n' % (' '*(start-1),
-                                        green('^' + '-'*length))
-                if len(source) > 2:  # write the middle lines
-                    for line in source[1:-1]:
-                        result += '  %s\n' % red("".join(line))
-                        result += '  %s\n' % green("-"*len(line))
-
-                # write the last line
-                result += '  %s\n' % red("".join(source[-1]))
-                result += '  %s\n' % green('-'*(end-1) + '^')
-
-        else:
-            result += '  File "%s", unknown location\n' % self.filename
-
-        result += yellow("%s: %s\n\n" %
-                         (self.__class__.__name__,
-                          self.message))
-
-        return result
+    """TypeError occurring during the normal use of Hy."""
 
 
-class HyMacroExpansionError(HyTypeError):
+class HyNameError(HyLanguageError, NameError):
+    """NameError occurring during the normal use of Hy."""
+
+
+class HyRequireError(HyLanguageError):
+    """Errors arising during the use of `require`
+
+    This, and any errors inheriting from this, are user-facing.
+    """
+
+
+class HyMacroExpansionError(HyLanguageError):
     """Errors caused by invalid use of Hy macros.
 
     This, and any errors inheriting from this, are user-facing.
@@ -158,97 +206,39 @@ class HyIOError(HyInternalError, IOError):
 class HySyntaxError(HyLanguageError, SyntaxError):
     """Error during the Lexing of a Hython expression."""
 
-    def __init__(self, message, filename=None, lineno=-1, colno=-1,
-                 source=None):
-        """
-        Parameters
-        ----------
-        message: str
-            The exception's message.
-        filename: str, optional
-            The filename for the source code generating this error.
-        lineno: int, optional
-            The line number of the error.
-        colno: int, optional
-            The column number of the error.
-        source: str, optional
-            The actual source code generating this error.
-        """
-        self.message = message
-        self.filename = filename
-        self.lineno = lineno
-        self.colno = colno
-        self.source = source
-        super(HySyntaxError, self).__init__(message,
-                                            # The builtin `SyntaxError` needs a
-                                            # tuple.
-                                            (filename, lineno, colno, source))
 
-    @staticmethod
-    def from_expression(message, expression, filename=None, source=None):
-        if not source:
-            # Maybe the expression object has its own source.
-            source = getattr(expression, 'source', None)
+def _module_filter_name(module_name):
+    try:
+        compiler_loader = pkgutil.get_loader(module_name)
+        if not compiler_loader:
+            return None
 
+        filename = compiler_loader.get_filename(module_name)
         if not filename:
-            filename = getattr(expression, 'filename', None)
+            return None
 
-        if source:
-            lineno = expression.start_line
-            colno = expression.start_column
-            end_line = getattr(expression, 'end_line', len(source))
-            lines = source.splitlines()
-            source = '\n'.join(lines[lineno-1:end_line])
+        if compiler_loader.is_package(module_name):
+            # Use the package directory (e.g. instead of `.../__init__.py`) so
+            # that we can filter all modules in a package.
+            return os.path.dirname(filename)
         else:
-            # We could attempt to extract the source given a filename, but we
-            # don't.
-            lineno = colno = -1
-
-        return HySyntaxError(message, filename, lineno, colno, source)
-
-    def __str__(self):
-        global _hy_colored_errors
-
-        output = traceback.format_exception_only(SyntaxError,
-                                                 SyntaxError(*self.args))
-
-        if _hy_colored_errors:
-            from hy.errors import colored
-            output[-1] = colored.yellow(output[-1])
-            if len(self.source) > 0:
-                output[-2] = colored.green(output[-2])
-                for line in output[::-2]:
-                    if line.strip().startswith(
-                            'File "{}", line'.format(self.filename)):
-                        break
-                output[-3] = colored.red(output[-3])
-
-        # Avoid "...expected str instance, ColoredString found"
-        return reduce(lambda x, y: x + y, output)
+            # Normalize filename endings, because tracebacks will use `pyc` when
+            # the loader says `py`.
+            return filename.replace('.pyc', '.py')
+    except Exception:
+        return None
 
 
-def _get_module_info(module):
-    compiler_loader = pkgutil.get_loader(module)
-    is_pkg = compiler_loader.is_package(module)
-    filename = compiler_loader.get_filename()
-    if is_pkg:
-        # Use package directory
-        return os.path.dirname(filename)
-    else:
-        # Normalize filename endings, because tracebacks will use `pyc` when
-        # the loader says `py`.
-        return filename.replace('.pyc', '.py')
+_tb_hidden_modules = {m for m in map(_module_filter_name,
+                                     ['hy.compiler', 'hy.lex',
+                                      'hy.cmdline', 'hy.lex.parser',
+                                      'hy.importer', 'hy._compat',
+                                      'hy.macros', 'hy.models',
+                                      'rply'])
+                      if m is not None}
 
 
-_tb_hidden_modules = {_get_module_info(m)
-                      for m in ['hy.compiler', 'hy.lex',
-                                'hy.cmdline', 'hy.lex.parser',
-                                'hy.importer', 'hy._compat',
-                                'hy.macros', 'hy.models',
-                                'rply']}
-
-
-def hy_exc_handler(exc_type, exc_value, exc_traceback):
+def hy_exc_filter(exc_type, exc_value, exc_traceback):
     """Produce exceptions print-outs with all frames originating from the
     modules in `_tb_hidden_modules` filtered out.
 
@@ -258,20 +248,33 @@ def hy_exc_handler(exc_type, exc_value, exc_traceback):
     This does not remove the frames from the actual tracebacks, so debugging
     will show everything.
     """
+    # frame = (filename, line number, function name*, text)
+    new_tb = []
+    for frame in traceback.extract_tb(exc_traceback):
+        if not (frame[0].replace('.pyc', '.py') in _tb_hidden_modules or
+                os.path.dirname(frame[0]) in _tb_hidden_modules):
+            new_tb += [frame]
+
+    lines = traceback.format_list(new_tb)
+
+    if lines:
+        lines.insert(0, "Traceback (most recent call last):\n")
+
+    lines.extend(traceback.format_exception_only(exc_type, exc_value))
+    output = ''.join(lines)
+
+    return output
+
+
+def hy_exc_handler(exc_type, exc_value, exc_traceback):
+    """A `sys.excepthook` handler that uses `hy_exc_filter` to
+    remove internal Hy frames from a traceback print-out.
+    """
+    if os.environ.get('HY_DEBUG', False):
+        return sys.__excepthook__(exc_type, exc_value, exc_traceback)
+
     try:
-        # frame = (filename, line number, function name*, text)
-        new_tb = [frame for frame in traceback.extract_tb(exc_traceback)
-                  if not (frame[0].replace('.pyc', '.py') in _tb_hidden_modules or
-                          os.path.dirname(frame[0]) in _tb_hidden_modules)]
-
-        lines = traceback.format_list(new_tb)
-
-        if lines:
-            lines.insert(0, "Traceback (most recent call last):\n")
-
-        lines.extend(traceback.format_exception_only(exc_type, exc_value))
-        output = ''.join(lines)
-
+        output = hy_exc_filter(exc_type, exc_value, exc_traceback)
         sys.stderr.write(output)
         sys.stderr.flush()
     except Exception:
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index f1465cd..eb3ac41 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -8,10 +8,9 @@ import re
 import sys
 import unicodedata
 
-from hy._compat import str_type, isidentifier, UCS4, reraise
+from hy._compat import str_type, isidentifier, UCS4
 from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
-from hy.errors import HySyntaxError
 
 try:
     from io import StringIO
@@ -35,15 +34,12 @@ def hy_parse(source, filename='<string>'):
     out : HyExpression
     """
     _source = re.sub(r'\A#!.*', '', source)
-    try:
-        res = HyExpression([HySymbol("do")] +
-                           tokenize(_source + "\n",
-                                    filename=filename))
-        res.source = source
-        res.filename = filename
-        return res
-    except HySyntaxError as e:
-        reraise(type(e), e, None)
+    res = HyExpression([HySymbol("do")] +
+                       tokenize(_source + "\n",
+                                filename=filename))
+    res.source = source
+    res.filename = filename
+    return res
 
 
 class ParserState(object):
@@ -70,8 +66,12 @@ def tokenize(source, filename=None):
                             state=ParserState(source, filename))
     except LexingError as e:
         pos = e.getsourcepos()
-        raise LexException("Could not identify the next token.", filename,
-                           pos.lineno, pos.colno, source)
+        raise LexException("Could not identify the next token.",
+                           None, filename, source,
+                           max(pos.lineno, 1),
+                           max(pos.colno, 1))
+    except LexException as e:
+        raise e
 
 
 mangle_delim = 'X'
diff --git a/hy/lex/exceptions.py b/hy/lex/exceptions.py
index fb9aa7a..449119a 100644
--- a/hy/lex/exceptions.py
+++ b/hy/lex/exceptions.py
@@ -8,26 +8,31 @@ class LexException(HySyntaxError):
 
     @classmethod
     def from_lexer(cls, message, state, token):
+        lineno = None
+        colno = None
+        source = state.source
         source_pos = token.getsourcepos()
-        if token.source_pos:
+
+        if source_pos:
             lineno = source_pos.lineno
             colno = source_pos.colno
+        elif source:
+            # Use the end of the last line of source for `PrematureEndOfInput`.
+            # We get rid of empty lines and spaces so that the error matches
+            # with the last piece of visible code.
+            lines = source.rstrip().splitlines()
+            lineno = lineno or len(lines)
+            colno = colno or len(lines[lineno - 1])
         else:
-            lineno = -1
-            colno = -1
+            lineno = lineno or 1
+            colno = colno or 1
 
-        if state.source:
-            lines = state.source.splitlines()
-            if lines[-1] == '':
-                del lines[-1]
-
-            if lineno < 1:
-                lineno = len(lines)
-            if colno < 1:
-                colno = len(lines[-1])
-
-            source = lines[lineno - 1]
-        return cls(message, state.filename, lineno, colno, source)
+        return cls(message,
+                   None,
+                   state.filename,
+                   source,
+                   lineno,
+                   colno)
 
 
 class PrematureEndOfInput(LexException):
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index f5cd5e5..c4df2a5 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -6,7 +6,6 @@
 from __future__ import unicode_literals
 
 from functools import wraps
-import re, unicodedata
 
 from rply import ParserGenerator
 
@@ -278,15 +277,6 @@ def symbol_like(obj):
 def error_handler(state, token):
     tokentype = token.gettokentype()
     if tokentype == '$end':
-        source_pos = token.source_pos or token.getsourcepos()
-        source = state.source
-        if source_pos:
-            lineno = source_pos.lineno
-            colno = source_pos.colno
-        else:
-            lineno = -1
-            colno = -1
-
         raise PrematureEndOfInput.from_lexer("Premature end of input", state,
                                              token)
     else:
diff --git a/hy/macros.py b/hy/macros.py
index 274702f..d668077 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -5,13 +5,15 @@ import sys
 import importlib
 import inspect
 import pkgutil
+import traceback
 
 from contextlib import contextmanager
 
-from hy._compat import PY3, string_types, reraise
+from hy._compat import PY3, string_types, reraise, rename_function
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
-from hy.errors import HyTypeError, HyMacroExpansionError
+from hy.errors import (HyLanguageError, HyMacroExpansionError, HyTypeError,
+                       HyRequireError)
 
 try:
     # Check if we have the newer inspect.signature available.
@@ -50,7 +52,7 @@ def macro(name):
     """
     name = mangle(name)
     def _(fn):
-        fn.__name__ = '({})'.format(name)
+        fn = rename_function(fn, name)
         try:
             fn._hy_macro_pass_compiler = has_kwargs(fn)
         except Exception:
@@ -75,7 +77,7 @@ def tag(name):
         if not PY3:
             _name = _name.encode('UTF-8')
 
-        fn.__name__ = _name
+        fn = rename_function(fn, _name)
 
         module = inspect.getmodule(fn)
 
@@ -150,7 +152,6 @@ def require(source_module, target_module, assignments, prefix=""):
     out: boolean
         Whether or not macros and tags were actually transferred.
     """
-
     if target_module is None:
         parent_frame = inspect.stack()[1][0]
         target_namespace = parent_frame.f_globals
@@ -161,7 +162,7 @@ def require(source_module, target_module, assignments, prefix=""):
     elif inspect.ismodule(target_module):
         target_namespace = target_module.__dict__
     else:
-        raise TypeError('`target_module` is not a recognized type: {}'.format(
+        raise HyTypeError('`target_module` is not a recognized type: {}'.format(
             type(target_module)))
 
     # Let's do a quick check to make sure the source module isn't actually
@@ -173,14 +174,17 @@ def require(source_module, target_module, assignments, prefix=""):
         return False
 
     if not inspect.ismodule(source_module):
-        source_module = importlib.import_module(source_module)
+        try:
+            source_module = importlib.import_module(source_module)
+        except ImportError as e:
+            reraise(HyRequireError, HyRequireError(e.args[0]), None)
 
     source_macros = source_module.__dict__.setdefault('__macros__', {})
     source_tags = source_module.__dict__.setdefault('__tags__', {})
 
     if len(source_module.__macros__) + len(source_module.__tags__) == 0:
         if assignments != "ALL":
-            raise ImportError('The module {} has no macros or tags'.format(
+            raise HyRequireError('The module {} has no macros or tags'.format(
                 source_module))
         else:
             return False
@@ -205,7 +209,7 @@ def require(source_module, target_module, assignments, prefix=""):
         elif _name in source_module.__tags__:
             target_tags[alias] = source_tags[_name]
         else:
-            raise ImportError('Could not require name {} from {}'.format(
+            raise HyRequireError('Could not require name {} from {}'.format(
                 _name, source_module))
 
     return True
@@ -239,50 +243,33 @@ def load_macros(module):
                                 if k not in module_tags})
 
 
-def make_empty_fn_copy(fn):
-    try:
-        # This might fail if fn has parameters with funny names, like o!n. In
-        # such a case, we return a generic function that ensures the program
-        # can continue running. Unfortunately, the error message that might get
-        # raised later on while expanding a macro might not make sense at all.
-
-        formatted_args = format_args(fn)
-        fn_str = 'lambda {}: None'.format(
-            formatted_args.lstrip('(').rstrip(')'))
-        empty_fn = eval(fn_str)
-
-    except Exception:
-
-        def empty_fn(*args, **kwargs):
-            None
-
-    return empty_fn
-
-
 @contextmanager
 def macro_exceptions(module, macro_tree, compiler=None):
     try:
         yield
+    except HyLanguageError as e:
+        # These are user-level Hy errors occurring in the macro.
+        # We want to pass them up to the user.
+        reraise(type(e), e, sys.exc_info()[2])
     except Exception as e:
-        try:
-            filename = inspect.getsourcefile(module)
-            source = inspect.getsource(module)
-        except TypeError:
-            if compiler:
-                filename = compiler.filename
-                source = compiler.source
 
-        if not isinstance(e, HyTypeError):
-            exc_type = HyMacroExpansionError
-            msg = "expanding `{}': ".format(macro_tree[0])
-            msg += str(e).replace("<lambda>()", "", 1).strip()
+        if compiler:
+            filename = compiler.filename
+            source = compiler.source
         else:
-            exc_type = HyTypeError
-            msg = e.message
+            filename = None
+            source = None
 
-        reraise(exc_type,
-                exc_type(msg, filename, macro_tree, source),
-                sys.exc_info()[2].tb_next)
+        exc_msg = '  '.join(traceback.format_exception_only(
+            sys.exc_info()[0], sys.exc_info()[1]))
+
+        msg = "expanding macro {}\n  ".format(str(macro_tree[0]))
+        msg += exc_msg
+
+        reraise(HyMacroExpansionError,
+                HyMacroExpansionError(
+                    msg, macro_tree, filename, source),
+                sys.exc_info()[2])
 
 
 def macroexpand(tree, module, compiler=None, once=False):
@@ -353,8 +340,6 @@ def macroexpand(tree, module, compiler=None, once=False):
             opts['compiler'] = compiler
 
         with macro_exceptions(module, tree, compiler):
-            m_copy = make_empty_fn_copy(m)
-            m_copy(module.__name__, *tree[1:], **opts)
             obj = m(module.__name__, *tree[1:], **opts)
 
         if isinstance(obj, HyExpression):
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 0c7bcd5..9311eef 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -6,9 +6,8 @@
 from __future__ import unicode_literals
 
 from hy import HyString
-from hy.models import HyObject
 from hy.compiler import hy_compile, hy_eval
-from hy.errors import HyCompileError, HyTypeError
+from hy.errors import HyCompileError, HyLanguageError, HyError
 from hy.lex import hy_parse
 from hy.lex.exceptions import LexException, PrematureEndOfInput
 from hy._compat import PY3
@@ -27,7 +26,7 @@ def _ast_spotcheck(arg, root, secondary):
 
 
 def can_compile(expr):
-    return hy_compile(hy_parse(expr), "__main__")
+    return hy_compile(hy_parse(expr), __name__)
 
 
 def can_eval(expr):
@@ -35,21 +34,16 @@ def can_eval(expr):
 
 
 def cant_compile(expr):
-    try:
-        hy_compile(hy_parse(expr), "__main__")
-        assert False
-    except HyTypeError as e:
+    with pytest.raises(HyError) as excinfo:
+        hy_compile(hy_parse(expr), __name__)
+
+    if issubclass(excinfo.type, HyLanguageError):
+        assert excinfo.value.msg
+        return excinfo.value
+    elif issubclass(excinfo.type, HyCompileError):
         # Anything that can't be compiled should raise a user friendly
         # error, otherwise it's a compiler bug.
-        assert isinstance(e.expression, HyObject)
-        assert e.message
-        return e
-    except HyCompileError as e:
-        # Anything that can't be compiled should raise a user friendly
-        # error, otherwise it's a compiler bug.
-        assert isinstance(e.exception, HyTypeError)
-        assert e.traceback
-        return e
+        return excinfo.value
 
 
 def s(x):
@@ -60,11 +54,9 @@ def test_ast_bad_type():
     "Make sure AST breakage can happen"
     class C:
         pass
-    try:
-        hy_compile(C(), "__main__")
-        assert True is False
-    except TypeError:
-        pass
+
+    with pytest.raises(TypeError):
+        hy_compile(C(), __name__, filename='<string>', source='')
 
 
 def test_empty_expr():
@@ -473,7 +465,7 @@ def test_lambda_list_keywords_kwonly():
         assert code.body[0].args.kw_defaults[1].n == 2
     else:
         exception = cant_compile(kwonly_demo)
-        assert isinstance(exception, HyTypeError)
+        assert isinstance(exception, HyLanguageError)
         message = exception.args[0]
         assert message == "&kwonly parameters require Python 3"
 
@@ -489,9 +481,9 @@ def test_lambda_list_keywords_mixed():
 
 def test_missing_keyword_argument_value():
     """Ensure the compiler chokes on missing keyword argument values."""
-    with pytest.raises(HyTypeError) as excinfo:
+    with pytest.raises(HyLanguageError) as excinfo:
         can_compile("((fn [x] x) :x)")
-    assert excinfo.value.message == "Keyword argument :x needs a value."
+    assert excinfo.value.msg == "Keyword argument :x needs a value."
 
 
 def test_ast_unicode_strings():
@@ -500,7 +492,7 @@ def test_ast_unicode_strings():
     def _compile_string(s):
         hy_s = HyString(s)
 
-        code = hy_compile([hy_s], "__main__")
+        code = hy_compile([hy_s], __name__, filename='<string>', source=s)
         # We put hy_s in a list so it isn't interpreted as a docstring.
 
         # code == ast.Module(body=[ast.Expr(value=ast.List(elts=[ast.Str(s=xxx)]))])
@@ -541,7 +533,7 @@ Only one leading newline should be removed.
 
 def test_compile_error():
     """Ensure we get compile error in tricky cases"""
-    with pytest.raises(HyTypeError) as excinfo:
+    with pytest.raises(HyLanguageError) as excinfo:
         can_compile("(fn [] (in [1 2 3]))")
 
 
@@ -549,11 +541,11 @@ def test_for_compile_error():
     """Ensure we get compile error in tricky 'for' cases"""
     with pytest.raises(PrematureEndOfInput) as excinfo:
         can_compile("(fn [] (for)")
-    assert excinfo.value.message == "Premature end of input"
+    assert excinfo.value.msg == "Premature end of input"
 
     with pytest.raises(LexException) as excinfo:
         can_compile("(fn [] (for)))")
-    assert excinfo.value.message == "Ran into a RPAREN where it wasn't expected."
+    assert excinfo.value.msg == "Ran into a RPAREN where it wasn't expected."
 
     cant_compile("(fn [] (for [x] x))")
 
@@ -605,13 +597,13 @@ def test_setv_builtins():
 
 
 def test_top_level_unquote():
-    with pytest.raises(HyTypeError) as excinfo:
+    with pytest.raises(HyLanguageError) as excinfo:
         can_compile("(unquote)")
-    assert excinfo.value.message == "The special form 'unquote' is not allowed here"
+    assert excinfo.value.msg == "The special form 'unquote' is not allowed here"
 
-    with pytest.raises(HyTypeError) as excinfo:
+    with pytest.raises(HyLanguageError) as excinfo:
         can_compile("(unquote-splice)")
-    assert excinfo.value.message == "The special form 'unquote-splice' is not allowed here"
+    assert excinfo.value.msg == "The special form 'unquote-splice' is not allowed here"
 
 
 def test_lots_of_comment_lines():
diff --git a/tests/macros/test_macro_processor.py b/tests/macros/test_macro_processor.py
index 309ca49..134bd5b 100644
--- a/tests/macros/test_macro_processor.py
+++ b/tests/macros/test_macro_processor.py
@@ -50,8 +50,7 @@ def test_preprocessor_exceptions():
     """ Test that macro expansion raises appropriate exceptions"""
     with pytest.raises(HyMacroExpansionError) as excinfo:
         macroexpand(tokenize('(defn)')[0], __name__, HyASTCompiler(__name__))
-    assert "_hy_anon_fn_" not in excinfo.value.message
-    assert "TypeError" not in excinfo.value.message
+    assert "_hy_anon_" not in excinfo.value.msg
 
 
 def test_macroexpand_nan():
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index a89eece..ab8eaf4 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -687,5 +687,5 @@ result['y in globals'] = 'y' in globals()")
   (doc doc)
   (setv out_err (.readouterr capsys))
   (assert (.startswith (.strip (first out_err))
-            "Help on function (doc) in module hy.core.macros:"))
+            "Help on function doc in module hy.core.macros:"))
   (assert (empty? (second out_err))))
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index c237a5a..65c629a 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -7,7 +7,7 @@
         [sys :as systest]
         re
         [operator [or_]]
-        [hy.errors [HyTypeError]]
+        [hy.errors [HyLanguageError]]
         pytest)
 (import sys)
 
@@ -68,16 +68,16 @@
   "NATIVE: test that setv doesn't work on names Python can't assign to
   and that we can't mangle"
   (try (eval '(setv None 1))
-       (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
   (try (eval '(defn None [] (print "hello")))
-       (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
   (when PY3
     (try (eval '(setv False 1))
-         (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+         (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
     (try (eval '(setv True 0))
-         (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))
+         (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
     (try (eval '(defn True [] (print "hello")))
-         (except [e [TypeError]] (assert (in "Can't assign to" (str e)))))))
+         (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))))
 
 
 (defn test-setv-pairs []
@@ -87,7 +87,7 @@
   (assert (= b 2))
   (setv y 0 x 1 y x)
   (assert (= y 1))
-  (with [(pytest.raises HyTypeError)]
+  (with [(pytest.raises HyLanguageError)]
     (eval '(setv a 1 b))))
 
 
@@ -144,29 +144,29 @@
     (do
       (eval '(setv (do 1 2) 1))
       (assert False))
-    (except [e HyTypeError]
-      (assert (= e.message "Can't assign or delete a non-expression"))))
+    (except [e HyLanguageError]
+      (assert (= e.msg "Can't assign or delete a non-expression"))))
 
   (try
     (do
       (eval '(setv 1 1))
       (assert False))
-    (except [e HyTypeError]
-      (assert (= e.message "Can't assign or delete a HyInteger"))))
+    (except [e HyLanguageError]
+      (assert (= e.msg "Can't assign or delete a HyInteger"))))
 
   (try
     (do
       (eval '(setv {1 2} 1))
       (assert False))
-    (except [e HyTypeError]
-      (assert (= e.message "Can't assign or delete a HyDict"))))
+    (except [e HyLanguageError]
+      (assert (= e.msg "Can't assign or delete a HyDict"))))
 
   (try
     (do
       (eval '(del 1 1))
       (assert False))
-    (except [e HyTypeError]
-      (assert (= e.message "Can't assign or delete a HyInteger")))))
+    (except [e HyLanguageError]
+      (assert (= e.msg "Can't assign or delete a HyInteger")))))
 
 
 (defn test-no-str-as-sym []
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 1700b5d..6759f42 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -3,7 +3,7 @@
 ;; license. See the LICENSE.
 
 (import pytest
-        [hy.errors [HyTypeError]])
+        [hy.errors [HyTypeError HyMacroExpansionError]])
 
 (defmacro rev [&rest body]
   "Execute the `body` statements in reverse"
@@ -66,13 +66,13 @@
   (try
     (eval '(defmacro f [&kwonly a b]))
     (except [e HyTypeError]
-      (assert (= e.message "macros cannot use &kwonly")))
+      (assert (= e.msg "macros cannot use &kwonly")))
     (else (assert False)))
 
   (try
     (eval '(defmacro f [&kwargs kw]))
     (except [e HyTypeError]
-      (assert (= e.message "macros cannot use &kwargs")))
+      (assert (= e.msg "macros cannot use &kwargs")))
     (else (assert False))))
 
 (defn test-fn-calling-macro []
@@ -483,3 +483,28 @@ in expansions."
 
   (test-macro)
   (assert (= blah 1)))
+
+
+(defn test-macro-errors []
+  (import traceback
+          [hy.importer [hy-parse]])
+
+  (setv test-expr (hy-parse "(defmacro blah [x] `(print ~@z)) (blah y)"))
+
+  (with [excinfo (pytest.raises HyMacroExpansionError)]
+    (eval test-expr))
+
+  (setv output (traceback.format_exception_only
+                 excinfo.type excinfo.value))
+  (setv output (cut (.splitlines (.strip (first output))) 1))
+
+  (setv expected ["  File \"<string>\", line 1"
+                  "    (defmacro blah [x] `(print ~@z)) (blah y)"
+                  "                                     ^------^"
+                  "expanding macro blah"
+                  "  NameError: global name 'z' is not defined"])
+
+  (assert (= (cut expected 0 -1) (cut output 0 -1)))
+  (assert (or (= (get expected -1) (get output -1))
+              ;; Handle PyPy's peculiarities
+              (= (.replace (get expected -1) "global " "") (get output -1)))))
diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy
index 716eb77..e08edbb 100644
--- a/tests/native_tests/operators.hy
+++ b/tests/native_tests/operators.hy
@@ -28,7 +28,7 @@
 (defmacro forbid [expr]
   `(assert (try
     (eval '~expr)
-    (except [TypeError] True)
+    (except [[TypeError SyntaxError]] True)
     (else (raise AssertionError)))))
 
 
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 8aef923..aad45e5 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -11,6 +11,7 @@ import shlex
 import subprocess
 
 from hy.importer import cache_from_source
+from hy._compat import PY3
 
 import pytest
 
@@ -123,7 +124,16 @@ def test_bin_hy_stdin_as_arrow():
 
 def test_bin_hy_stdin_error_underline_alignment():
     _, err = run_cmd("hy", "(defmacro mabcdefghi [x] x)\n(mabcdefghi)")
-    assert "\n  (mabcdefghi)\n  ^----------^" in err
+
+    msg_idx = err.rindex("    (mabcdefghi)")
+    assert msg_idx
+    err_parts = err[msg_idx:].splitlines()
+    assert err_parts[1].startswith("    ^----------^")
+    assert err_parts[2].startswith("expanding macro mabcdefghi")
+    assert (err_parts[3].startswith("  TypeError: mabcdefghi") or
+            # PyPy can use a function's `__name__` instead of
+            # `__code__.co_name`.
+            err_parts[3].startswith("  TypeError: (mabcdefghi)"))
 
 
 def test_bin_hy_stdin_except_do():
@@ -153,6 +163,62 @@ def test_bin_hy_stdin_unlocatable_hytypeerror():
     assert "AZ" in err
 
 
+def test_bin_hy_error_parts_length():
+    """Confirm that exception messages print arrows surrounding the affected
+    expression."""
+    prg_str = """
+    (import hy.errors
+            [hy.importer [hy-parse]])
+
+    (setv test-expr (hy-parse "(+ 1\n\n'a 2 3\n\n 1)"))
+    (setv test-expr.start-line {})
+    (setv test-expr.start-column {})
+    (setv test-expr.end-column {})
+
+    (raise (hy.errors.HyLanguageError
+             "this\nis\na\nmessage"
+             test-expr
+             None
+             None))
+    """
+
+    # Up-arrows right next to each other.
+    _, err = run_cmd("hy", prg_str.format(3, 1, 2))
+
+    msg_idx = err.rindex("HyLanguageError:")
+    assert msg_idx
+    err_parts = err[msg_idx:].splitlines()[1:]
+
+    expected = ['  File "<string>", line 3',
+                '    \'a 2 3',
+                '    ^^',
+                'this',
+                'is',
+                'a',
+                'message']
+
+    for obs, exp in zip(err_parts, expected):
+        assert obs.startswith(exp)
+
+    # Make sure only one up-arrow is printed
+    _, err = run_cmd("hy", prg_str.format(3, 1, 1))
+
+    msg_idx = err.rindex("HyLanguageError:")
+    assert msg_idx
+    err_parts = err[msg_idx:].splitlines()[1:]
+    assert err_parts[2] == '    ^'
+
+    # Make sure lines are printed in between arrows separated by more than one
+    # character.
+    _, err = run_cmd("hy", prg_str.format(3, 1, 6))
+    print(err)
+
+    msg_idx = err.rindex("HyLanguageError:")
+    assert msg_idx
+    err_parts = err[msg_idx:].splitlines()[1:]
+    assert err_parts[2] == '    ^----^'
+
+
 def test_bin_hy_stdin_bad_repr():
     # https://github.com/hylang/hy/issues/1389
     output, err = run_cmd("hy", """
@@ -423,3 +489,86 @@ def test_bin_hy_macro_require():
     assert os.path.exists(cache_from_source(test_file))
     output, _ = run_cmd("hy {}".format(test_file))
     assert "abc" == output.strip()
+
+
+def test_bin_hy_tracebacks():
+    """Make sure the printed tracebacks are correct."""
+
+    # We want the filtered tracebacks.
+    os.environ['HY_DEBUG'] = ''
+
+    def req_err(x):
+        assert x == '{}HyRequireError: No module named {}'.format(
+            'hy.errors.' if PY3 else '',
+            (repr if PY3 else str)('not_a_real_module'))
+
+    # Modeled after
+    #   > python -c 'import not_a_real_module'
+    #   Traceback (most recent call last):
+    #     File "<string>", line 1, in <module>
+    #   ImportError: No module named not_a_real_module
+    _, error = run_cmd('hy', '(require not-a-real-module)')
+    error_lines = error.splitlines()
+    if error_lines[-1] == '':
+        del error_lines[-1]
+    assert len(error_lines) <= 10
+      # Rough check for the internal traceback filtering
+    req_err(error_lines[4 if PY3 else -1])
+
+    _, error = run_cmd('hy -c "(require not-a-real-module)"', expect=1)
+    error_lines = error.splitlines()
+    assert len(error_lines) <= 4
+    req_err(error_lines[-1])
+
+    output, error = run_cmd('hy -i "(require not-a-real-module)"')
+    assert output.startswith('=> ')
+    print(error.splitlines())
+    req_err(error.splitlines()[3 if PY3 else -3])
+
+    # Modeled after
+    #   > python -c 'print("hi'
+    #     File "<string>", line 1
+    #       print("hi
+    #               ^
+    #   SyntaxError: EOL while scanning string literal
+    _, error = run_cmd(r'hy -c "(print \""', expect=1)
+    peoi = (
+        '  File "<string>", line 1\n'
+        '    (print "\n'
+        '           ^\n' +
+        '{}PrematureEndOfInput: Partial string literal\n'.format(
+            'hy.lex.exceptions.' if PY3 else ''))
+    assert error == peoi
+
+    # Modeled after
+    #   > python -i -c "print('"
+    #     File "<string>", line 1
+    #       print('
+    #             ^
+    #   SyntaxError: EOL while scanning string literal
+    #   >>>
+    output, error = run_cmd(r'hy -i "(print \""')
+    assert output.startswith('=> ')
+    assert error.startswith(peoi)
+
+    # Modeled after
+    #   > python -c 'print(a)'
+    #   Traceback (most recent call last):
+    #     File "<string>", line 1, in <module>
+    #   NameError: name 'a' is not defined
+    output, error = run_cmd('hy -c "(print a)"', expect=1)
+    error_lines = error.splitlines()
+    assert error_lines[3] == '  File "<string>", line 1, in <module>'
+    # PyPy will add "global" to this error message, so we work around that.
+    assert error_lines[-1].strip().replace(' global', '') == (
+        "NameError: name 'a' is not defined")
+
+    # Modeled after
+    #   > python -c 'compile()'
+    #   Traceback (most recent call last):
+    #     File "<string>", line 1, in <module>
+    #   TypeError: Required argument 'source' (pos 1) not found
+    output, error = run_cmd('hy -c "(compile)"', expect=1)
+    error_lines = error.splitlines()
+    assert error_lines[-2] == '  File "<string>", line 1, in <module>'
+    assert error_lines[-1].startswith('TypeError')
diff --git a/tests/test_lex.py b/tests/test_lex.py
index b0a03dc..f709719 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -61,7 +61,7 @@ def test_lex_single_quote_err():
     with lexe() as execinfo:
         tokenize("' ")
     check_ex(execinfo, [
-        '  File "<string>", line -1\n',
+        '  File "<string>", line 1\n',
         "    '\n",
         '    ^\n',
         'LexException: Could not identify the next token.\n'])
@@ -472,7 +472,7 @@ def test_lex_exception_filtering(capsys):
 
     # First, test for PrematureEndOfInput
     with peoi() as execinfo:
-        tokenize(" \n (foo")
+        tokenize(" \n (foo\n       \n")
     check_trace_output(capsys, execinfo, [
         '  File "<string>", line 2',
         '    (foo',

From 4ae4baac2a0fd898cd83345231287f9795aa6f8c Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Thu, 1 Nov 2018 16:40:13 -0500
Subject: [PATCH 104/223] Cache command line source for exceptions

Source entered interactively can now be displayed in traceback output.  Also,
the REPL object is now available in its namespace, so that, for instance,
display options--like `spy`--can be turned on and off interactively.

Closes hylang/hy#1397.
---
 hy/_compat.py     |   5 +-
 hy/cmdline.py     | 171 ++++++++++++++++++++++++++++++++++++++++++----
 hy/errors.py      |   3 +-
 tests/test_bin.py |  20 +++---
 4 files changed, 168 insertions(+), 31 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 2711445..92aa392 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -28,10 +28,7 @@ string_types = str   if PY3 else basestring   # NOQA
 if PY3:
     raise_src = textwrap.dedent('''
     def raise_from(value, from_value):
-        try:
-            raise value from from_value
-        finally:
-            traceback = None
+        raise value from from_value
     ''')
 
     def reraise(exc_type, value, traceback=None):
diff --git a/hy/cmdline.py b/hy/cmdline.py
index a9f3af3..f65379d 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -15,13 +15,20 @@ import py_compile
 import traceback
 import runpy
 import types
+import time
+import linecache
+import hashlib
+import codeop
 
 import astor.code_gen
 
 import hy
+
 from hy.lex import hy_parse, mangle
+from contextlib import contextmanager
 from hy.lex.exceptions import PrematureEndOfInput
-from hy.compiler import HyASTCompiler, hy_eval, hy_compile, ast_compile
+from hy.compiler import (HyASTCompiler, hy_eval, hy_compile,
+                         hy_ast_compile_flags)
 from hy.errors import (HyLanguageError, HyRequireError, HyMacroExpansionError,
                        filtered_hy_exceptions, hy_exc_handler)
 from hy.importer import runhy
@@ -31,6 +38,11 @@ from hy.models import HyExpression, HyString, HySymbol
 from hy._compat import builtins, PY3, FileNotFoundError
 
 
+sys.last_type = None
+sys.last_value = None
+sys.last_traceback = None
+
+
 class HyQuitter(object):
     def __init__(self, name):
         self.name = name
@@ -51,14 +63,101 @@ class HyQuitter(object):
 builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
+@contextmanager
+def extend_linecache(add_cmdline_cache):
+    _linecache_checkcache = linecache.checkcache
 
-class HyCommandCompiler(object):
-    def __init__(self, module, ast_callback=None, hy_compiler=None):
+    def _cmdline_checkcache(*args):
+        _linecache_checkcache(*args)
+        linecache.cache.update(add_cmdline_cache)
+
+    linecache.checkcache = _cmdline_checkcache
+    yield
+    linecache.checkcache = _linecache_checkcache
+
+
+_codeop_maybe_compile = codeop._maybe_compile
+
+
+def _hy_maybe_compile(compiler, source, filename, symbol):
+    """The `codeop` version of this will compile the same source multiple
+    times, and, since we have macros and things like `eval-and-compile`, we
+    can't allow that.
+    """
+    if not isinstance(compiler, HyCompile):
+        return _codeop_maybe_compile(compiler, source, filename, symbol)
+
+    for line in source.split("\n"):
+        line = line.strip()
+        if line and line[0] != ';':
+            # Leave it alone (could do more with Hy syntax)
+            break
+    else:
+        if symbol != "eval":
+            # Replace it with a 'pass' statement (i.e. tell the compiler to do
+            # nothing)
+            source = "pass"
+
+    return compiler(source, filename, symbol)
+
+
+codeop._maybe_compile = _hy_maybe_compile
+
+
+class HyCompile(codeop.Compile, object):
+    """This compiler uses `linecache` like
+    `IPython.core.compilerop.CachingCompiler`.
+    """
+
+    def __init__(self, module, locals, ast_callback=None,
+                 hy_compiler=None, cmdline_cache={}):
         self.module = module
+        self.locals = locals
         self.ast_callback = ast_callback
         self.hy_compiler = hy_compiler
 
+        super(HyCompile, self).__init__()
+
+        self.flags |= hy_ast_compile_flags
+
+        self.cmdline_cache = cmdline_cache
+
+    def _cache(self, source, name):
+        entry = (len(source),
+                 time.time(),
+                 [line + '\n' for line in source.splitlines()],
+                 name)
+
+        linecache.cache[name] = entry
+        self.cmdline_cache[name] = entry
+
+    def _update_exc_info(self):
+            self.locals['_hy_last_type'] = sys.last_type
+            self.locals['_hy_last_value'] = sys.last_value
+            # Skip our frame.
+            sys.last_traceback = getattr(sys.last_traceback, 'tb_next',
+                                         sys.last_traceback)
+            self.locals['_hy_last_traceback'] = sys.last_traceback
+
     def __call__(self, source, filename="<input>", symbol="single"):
+
+        if source == 'pass':
+            # We need to return a no-op to signal that no more input is needed.
+            return (compile(source, filename, symbol),) * 2
+
+        hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest()
+        name = '{}-{}'.format(filename.strip('<>'), hash_digest)
+
+        try:
+            hy_ast = hy_parse(source, filename=name)
+        except Exception:
+            # Capture a traceback without the compiler/REPL frames.
+            sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+            self._update_exc_info()
+            raise
+
+        self._cache(source, name)
+
         try:
             hy_ast = hy_parse(source, filename=filename)
             root_ast = ast.Interactive if symbol == 'single' else ast.Module
@@ -75,14 +174,39 @@ class HyCommandCompiler(object):
             if self.ast_callback:
                 self.ast_callback(exec_ast, eval_ast)
 
-            exec_code = ast_compile(exec_ast, filename, symbol)
-            eval_code = ast_compile(eval_ast, filename, 'eval')
+            exec_code = super(HyCompile, self).__call__(exec_ast, name, symbol)
+            eval_code = super(HyCompile, self).__call__(eval_ast, name, 'eval')
 
-            return exec_code, eval_code
-        except PrematureEndOfInput:
-            # Save these so that we can reraise/display when an incomplete
-            # interactive command is given at the prompt.
+        except HyLanguageError:
+            # Hy will raise exceptions during compile-time that Python would
+            # raise during run-time (e.g. import errors for `require`).  In
+            # order to work gracefully with the Python world, we convert such
+            # Hy errors to code that purposefully reraises those exceptions in
+            # the places where Python code expects them.
             sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+            self._update_exc_info()
+            exec_code = super(HyCompile, self).__call__(
+                'import hy._compat; hy._compat.reraise('
+                '_hy_last_type, _hy_last_value, _hy_last_traceback)',
+                name, symbol)
+            eval_code = super(HyCompile, self).__call__('None', name, 'eval')
+
+        return exec_code, eval_code
+
+
+class HyCommandCompiler(codeop.CommandCompiler, object):
+    def __init__(self, *args, **kwargs):
+        self.compiler = HyCompile(*args, **kwargs)
+
+    def __call__(self, *args, **kwargs):
+        try:
+            return super(HyCommandCompiler, self).__call__(*args, **kwargs)
+        except PrematureEndOfInput:
+            # We have to do this here, because `codeop._maybe_compile` won't
+            # take `None` for a return value (at least not in Python 2.7) and
+            # this exception type is also a `SyntaxError`, so it will be caught
+            # by `code.InteractiveConsole` base methods before it reaches our
+            # `runsource`.
             return None
 
 
@@ -111,11 +235,16 @@ class HyREPL(code.InteractiveConsole, object):
 
         self.hy_compiler = HyASTCompiler(self.module)
 
-        self.compile = HyCommandCompiler(self.module, self.ast_callback,
-                                         self.hy_compiler)
+        self.cmdline_cache = {}
+        self.compile = HyCommandCompiler(self.module,
+                                         self.locals,
+                                         ast_callback=self.ast_callback,
+                                         hy_compiler=self.hy_compiler,
+                                         cmdline_cache=self.cmdline_cache)
 
         self.spy = spy
         self.last_value = None
+        self.print_last_value = True
 
         if output_fn is None:
             self.output_fn = repr
@@ -133,6 +262,9 @@ class HyREPL(code.InteractiveConsole, object):
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
 
+        # Allow access to the running REPL instance
+        self.locals['_hy_repl'] = self
+
     def ast_callback(self, exec_ast, eval_ast):
         if self.spy:
             try:
@@ -146,11 +278,17 @@ class HyREPL(code.InteractiveConsole, object):
                     traceback.format_exc())
                 self.write(msg)
 
-    def _error_wrap(self, error_fn, *args, **kwargs):
+    def _error_wrap(self, error_fn, exc_info_override=False, *args, **kwargs):
         sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
 
-        # Sadly, this method in Python 2.7 ignores an overridden
-        # `sys.excepthook`.
+        if exc_info_override:
+            # Use a traceback that doesn't have the REPL frames.
+            sys.last_type = self.locals.get('_hy_last_type', sys.last_type)
+            sys.last_value = self.locals.get('_hy_last_value', sys.last_value)
+            sys.last_traceback = self.locals.get('_hy_last_traceback',
+                                                 sys.last_traceback)
+
+        # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`.
         if sys.excepthook is sys.__excepthook__:
             error_fn(*args, **kwargs)
         else:
@@ -163,6 +301,7 @@ class HyREPL(code.InteractiveConsole, object):
             filename = self.filename
 
         self._error_wrap(super(HyREPL, self).showsyntaxerror,
+                         exc_info_override=True,
                          filename=filename)
 
     def showtraceback(self):
@@ -289,7 +428,9 @@ def run_repl(hr=None, **kwargs):
         hr = HyREPL(**kwargs)
 
     namespace = hr.locals
-    with filtered_hy_exceptions(), completion(Completer(namespace)):
+    with filtered_hy_exceptions(), \
+         extend_linecache(hr.cmdline_cache), \
+         completion(Completer(namespace)):
         hr.interact("{appname} {version} using "
                     "{py}({build}) {pyversion} on {os}".format(
                         appname=hy.__appname__,
diff --git a/hy/errors.py b/hy/errors.py
index 0579e96..a0cd589 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -257,8 +257,7 @@ def hy_exc_filter(exc_type, exc_value, exc_traceback):
 
     lines = traceback.format_list(new_tb)
 
-    if lines:
-        lines.insert(0, "Traceback (most recent call last):\n")
+    lines.insert(0, "Traceback (most recent call last):\n")
 
     lines.extend(traceback.format_exception_only(exc_type, exc_value))
     output = ''.join(lines)
diff --git a/tests/test_bin.py b/tests/test_bin.py
index aad45e5..06b3af9 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -6,7 +6,6 @@
 
 import os
 import re
-import sys
 import shlex
 import subprocess
 
@@ -523,7 +522,7 @@ def test_bin_hy_tracebacks():
     output, error = run_cmd('hy -i "(require not-a-real-module)"')
     assert output.startswith('=> ')
     print(error.splitlines())
-    req_err(error.splitlines()[3 if PY3 else -3])
+    req_err(error.splitlines()[2 if PY3 else -3])
 
     # Modeled after
     #   > python -c 'print("hi'
@@ -532,13 +531,14 @@ def test_bin_hy_tracebacks():
     #               ^
     #   SyntaxError: EOL while scanning string literal
     _, error = run_cmd(r'hy -c "(print \""', expect=1)
-    peoi = (
-        '  File "<string>", line 1\n'
-        '    (print "\n'
-        '           ^\n' +
-        '{}PrematureEndOfInput: Partial string literal\n'.format(
-            'hy.lex.exceptions.' if PY3 else ''))
-    assert error == peoi
+    peoi_re = (
+        r'Traceback \(most recent call last\):\n'
+        r'  File "(?:<string>|string-[0-9a-f]+)", line 1\n'
+        r'    \(print "\n'
+        r'           \^\n' +
+        r'{}PrematureEndOfInput: Partial string literal\n'.format(
+            r'hy\.lex\.exceptions\.' if PY3 else ''))
+    assert re.search(peoi_re, error)
 
     # Modeled after
     #   > python -i -c "print('"
@@ -549,7 +549,7 @@ def test_bin_hy_tracebacks():
     #   >>>
     output, error = run_cmd(r'hy -i "(print \""')
     assert output.startswith('=> ')
-    assert error.startswith(peoi)
+    assert re.match(peoi_re, error)
 
     # Modeled after
     #   > python -c 'print(a)'

From 9e62903d8aa8a22f18bd58790762ec63b396dcde Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Tue, 13 Nov 2018 12:11:51 -0600
Subject: [PATCH 105/223] Add special exception and handling for wrapper errors

---
 hy/errors.py                        |  9 +++++++++
 hy/macros.py                        |  6 +++---
 hy/models.py                        |  3 ++-
 tests/native_tests/native_macros.hy | 31 +++++++++++++++++++----------
 4 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/hy/errors.py b/hy/errors.py
index a0cd589..0b7619e 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -207,6 +207,15 @@ class HySyntaxError(HyLanguageError, SyntaxError):
     """Error during the Lexing of a Hython expression."""
 
 
+class HyWrapperError(HyError, TypeError):
+    """Errors caused by language model object wrapping.
+
+    These can be caused by improper user-level use of a macro, so they're
+    not really "internal".  If they arise due to anything else, they're an
+    internal/compiler problem, though.
+    """
+
+
 def _module_filter_name(module_name):
     try:
         compiler_loader = pkgutil.get_loader(module_name)
diff --git a/hy/macros.py b/hy/macros.py
index d668077..e2cec31 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -342,10 +342,10 @@ def macroexpand(tree, module, compiler=None, once=False):
         with macro_exceptions(module, tree, compiler):
             obj = m(module.__name__, *tree[1:], **opts)
 
-        if isinstance(obj, HyExpression):
-            obj.module = inspect.getmodule(m)
+            if isinstance(obj, HyExpression):
+                obj.module = inspect.getmodule(m)
 
-        tree = replace_hy_obj(obj, tree)
+            tree = replace_hy_obj(obj, tree)
 
         if once:
             break
diff --git a/hy/models.py b/hy/models.py
index 134e322..478c691 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -7,6 +7,7 @@ from contextlib import contextmanager
 from math import isnan, isinf
 from hy import _initialize_env_var
 from hy._compat import PY3, str_type, bytes_type, long_type, string_types
+from hy.errors import HyWrapperError
 from fractions import Fraction
 from clint.textui import colored
 
@@ -64,7 +65,7 @@ def wrap_value(x):
 
     new = _wrappers.get(type(x), lambda y: y)(x)
     if not isinstance(new, HyObject):
-        raise TypeError("Don't know how to wrap {!r}: {!r}".format(type(x), x))
+        raise HyWrapperError("Don't know how to wrap {!r}: {!r}".format(type(x), x))
     if isinstance(x, HyObject):
         new = new.replace(x, recursive=False)
     if not hasattr(new, "start_column"):
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 6759f42..d5c48c7 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -162,8 +162,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) __name__))
+  (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   ;; and make sure there is something new that starts with _;G|
@@ -189,8 +189,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) __name__))
+  (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;a|") s1))
@@ -213,8 +213,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) __name__))
+  (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;res|") s1))
@@ -224,7 +224,7 @@
   ;; defmacro/g! didn't like numbers initially because they
   ;; don't have a startswith method and blew up during expansion
   (setv macro2 "(defmacro/g! two-point-zero [] `(+ (float 1) 1.0))")
-  (assert (hy-compile (hy-parse macro2) "foo")))
+  (assert (hy-compile (hy-parse macro2) __name__)))
 
 (defn test-defmacro! []
   ;; defmacro! must do everything defmacro/g! can
@@ -243,8 +243,8 @@
     ")
   ;; expand the macro twice, should use a different
   ;; gensym each time
-  (setv _ast1 (hy-compile (hy-parse macro1) "foo"))
-  (setv _ast2 (hy-compile (hy-parse macro1) "foo"))
+  (setv _ast1 (hy-compile (hy-parse macro1) __name__))
+  (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
   (assert (in (mangle "_;res|") s1))
@@ -254,7 +254,7 @@
   ;; defmacro/g! didn't like numbers initially because they
   ;; don't have a startswith method and blew up during expansion
   (setv macro2 "(defmacro! two-point-zero [] `(+ (float 1) 1.0))")
-  (assert (hy-compile (hy-parse macro2) "foo"))
+  (assert (hy-compile (hy-parse macro2) __name__))
 
   (defmacro! foo! [o!foo] `(do ~g!foo ~g!foo))
   ;; test that o! becomes g!
@@ -507,4 +507,13 @@ in expansions."
   (assert (= (cut expected 0 -1) (cut output 0 -1)))
   (assert (or (= (get expected -1) (get output -1))
               ;; Handle PyPy's peculiarities
-              (= (.replace (get expected -1) "global " "") (get output -1)))))
+              (= (.replace (get expected -1) "global " "") (get output -1))))
+
+
+  ;; This should throw a `HyWrapperError` that gets turned into a
+  ;; `HyMacroExpansionError`.
+  (with [excinfo (pytest.raises HyMacroExpansionError)]
+    (eval '(do (defmacro wrap-error-test []
+                 (fn []))
+               (wrap-error-test))))
+  (assert (in "HyWrapperError" (str excinfo.value))))

From 96d1b9c3face94d62711a6215a72722530ac588d Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 7 Feb 2019 13:50:30 -0500
Subject: [PATCH 106/223] Update README

---
 NEWS.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/NEWS.rst b/NEWS.rst
index 6f3d53c..20d45ef 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -21,6 +21,7 @@ New Features
 
 Bug Fixes
 ------------------------------
+* Cleaned up syntax and compiler errors
 * Fixed issue with empty arguments in `defmain`.
 * `require` now compiles to Python AST.
 * Fixed circular `require`s.

From 6769dda4b5e54cba161d33f94803cc6a06b96f63 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 7 Feb 2019 14:06:58 -0500
Subject: [PATCH 107/223] Clean up NEWS

---
 NEWS.rst | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 20d45ef..7c66102 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1,6 +1,6 @@
 .. default-role:: code
 
-Unreleased
+0.16.0
 ==============================
 
 Removals
@@ -9,35 +9,36 @@ Removals
 
 New Features
 ------------------------------
-* `eval` / `hy_eval` and `hy_compile` now accept an optional `compiler` argument
-  that enables the use of an existing `HyASTCompiler` instance.
+* `eval` / `hy_eval` and `hy_compile` now accept an optional `compiler`
+  argument that enables the use of an existing `HyASTCompiler` instance.
 * Keyword objects (not just literal keywords) can be called, as
   shorthand for `(get obj :key)`, and they accept a default value
   as a second argument.
-* Minimal macro expansion namespacing has been implemented.  As a result,
-  external macros no longer have to `require` their own macro dependencies.
+* Minimal macro expansion namespacing has been implemented. As a result,
+  external macros no longer have to `require` their own macro
+  dependencies.
 * Macros and tags now reside in module-level `__macros__` and `__tags__`
   attributes.
 
 Bug Fixes
 ------------------------------
-* Cleaned up syntax and compiler errors
-* Fixed issue with empty arguments in `defmain`.
+* Cleaned up syntax and compiler errors.
+* You can now call `defmain` with an empty lambda list.
 * `require` now compiles to Python AST.
-* Fixed circular `require`s.
+* Fixed circular `require`\s.
 * Fixed module reloading.
 * Fixed circular imports.
+* Fixed errors from `from __future__ import ...` statements and missing
+  Hy module docstrings caused by automatic importing of Hy builtins.
 * Fixed `__main__` file execution.
 * Fixed bugs in the handling of unpacking forms in method calls and
   attribute access.
 * Fixed crashes on Windows when calling `hy-repr` on date and time
   objects.
-* Fixed errors from `from __future__ import ...` statements and missing Hy
-  module docstrings caused by automatic importing of Hy builtins.
-* Fixed crash in `mangle` for some pathological inputs
-* Fixed incorrect mangling of some characters at low code points
-* Fixed a crash on certain versions of Python 2 due to changes
-  in the standard module `tokenize`
+* Fixed a crash in `mangle` for some pathological inputs.
+* Fixed incorrect mangling of some characters at low code points.
+* Fixed a crash on certain versions of Python 2 due to changes in the
+  standard module `tokenize`.
 
 0.15.0
 ==============================

From 247e64950de7f96d239e0136b253413d47463ec2 Mon Sep 17 00:00:00 2001
From: digikar99 <shubhamkarayare@gmail.com>
Date: Tue, 19 Feb 2019 18:39:24 +0530
Subject: [PATCH 108/223] Clean up the documentation of `defclass`

---
 docs/language/api.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index 4f52418..ab39770 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -415,14 +415,16 @@ They can be used to assign multiple variables at once:
 defclass
 --------
 
-New classes are declared with ``defclass``. It can takes two optional parameters:
-a vector defining a possible super classes and another vector containing
-attributes of the new class as two item vectors.
+New classes are declared with ``defclass``. It can take three optional parameters in the following order:
+a list defining (a) possible super class(es), a string (:term:`py:docstring`) and another list containing
+attributes of the new class along with their corresponding values.
 
 .. code-block:: clj
 
     (defclass class-name [super-class-1 super-class-2]
-      [attribute value]
+      "docstring"
+      [attribute1 value1
+       attribute2 value2]
 
       (defn method [self] (print "hello!")))
 

From d312dd5df2c4c2b470c7c1585fc58ac28dbcaf72 Mon Sep 17 00:00:00 2001
From: digikar99 <shubhamkarayare@gmail.com>
Date: Tue, 19 Feb 2019 19:51:17 +0530
Subject: [PATCH 109/223] Fix a ReST underline

---
 docs/language/api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index ab39770..c406679 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -239,7 +239,7 @@ as the user enters *k*.
 
 
 comment
-----
+-------
 
 The ``comment`` macro ignores its body and always expands to ``None``.
 Unlike linewise comments, the body of the ``comment`` macro must

From a338e4c323fd5f446c92d7cd50607c08abf0ad56 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 18 Feb 2019 11:56:25 -0500
Subject: [PATCH 110/223] Clean up a lexing test

---
 tests/test_lex.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/tests/test_lex.py b/tests/test_lex.py
index f709719..99658c6 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -240,22 +240,18 @@ def test_lex_bad_attrs():
     with lexe(): tokenize(":hello.foo")
 
 
-def test_lex_line_counting():
-    """ Make sure we can count lines / columns """
+def test_lex_column_counting():
     entry = tokenize("(foo (one two))")[0]
-
     assert entry.start_line == 1
     assert entry.start_column == 1
-
     assert entry.end_line == 1
     assert entry.end_column == 15
 
-    entry = entry[1]
-    assert entry.start_line == 1
-    assert entry.start_column == 6
-
-    assert entry.end_line == 1
-    assert entry.end_column == 14
+    inner_expr = entry[1]
+    assert inner_expr.start_line == 1
+    assert inner_expr.start_column == 6
+    assert inner_expr.end_line == 1
+    assert inner_expr.end_column == 14
 
 
 def test_lex_line_counting_multi():

From 997321d31cbe28ee68150518096efe194cc7d6a2 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 18 Feb 2019 12:00:38 -0500
Subject: [PATCH 111/223] Fix .end_line and .end_column of single-token models

---
 hy/lex/parser.py  |  7 +++++--
 hy/models.py      |  5 +++++
 tests/test_lex.py | 20 ++++++++++++++++++++
 3 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index c4df2a5..27180c5 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -31,8 +31,11 @@ def set_boundaries(fun):
             ret.end_line = end.lineno
             ret.end_column = end.colno
         else:
-            ret.end_line = start.lineno
-            ret.end_column = start.colno + len(p[0].value)
+            v = p[0].value
+            ret.end_line = start.lineno + v.count('\n')
+            ret.end_column = (len(v) - v.rindex('\n') - 1
+                if '\n' in v
+                else start.colno + len(v) - 1)
         return ret
     return wrapped
 
diff --git a/hy/models.py b/hy/models.py
index 478c691..ef51a26 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -33,6 +33,11 @@ class HyObject(object):
     """
     Generic Hy Object model. This is helpful to inject things into all the
     Hy lexing Objects at once.
+
+    The position properties (`start_line`, `end_line`, `start_column`,
+    `end_column`) are each 1-based and inclusive. For example, a symbol
+    `abc` starting at the first column would have `start_column` 1 and
+    `end_column` 3.
     """
     __properties__ = ["module", "start_line", "end_line", "start_column",
                       "end_column"]
diff --git a/tests/test_lex.py b/tests/test_lex.py
index 99658c6..304f69a 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -247,6 +247,12 @@ def test_lex_column_counting():
     assert entry.end_line == 1
     assert entry.end_column == 15
 
+    symbol = entry[0]
+    assert symbol.start_line == 1
+    assert symbol.start_column == 2
+    assert symbol.end_line == 1
+    assert symbol.end_column == 4
+
     inner_expr = entry[1]
     assert inner_expr.start_line == 1
     assert inner_expr.start_column == 6
@@ -254,6 +260,20 @@ def test_lex_column_counting():
     assert inner_expr.end_column == 14
 
 
+def test_lex_column_counting_with_literal_newline():
+    string, symbol = tokenize('"apple\nblueberry" abc')
+
+    assert string.start_line == 1
+    assert string.start_column == 1
+    assert string.end_line == 2
+    assert string.end_column == 10
+
+    assert symbol.start_line == 2
+    assert symbol.start_column == 12
+    assert symbol.end_line == 2
+    assert symbol.end_column == 14
+
+
 def test_lex_line_counting_multi():
     """ Make sure we can do multi-line tokenization """
     entries = tokenize("""

From 56f51a9a20c9393decb79060dfebd53f3694c90d Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 26 Feb 2019 13:56:42 -0500
Subject: [PATCH 112/223] Implement hy.lex.parse_one_thing

---
 hy/lex/__init__.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index eb3ac41..d133f5f 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -74,6 +74,33 @@ def tokenize(source, filename=None):
         raise e
 
 
+def parse_one_thing(src_string):
+    """Parse the first form from the string. Return it and the
+    remainder of the string."""
+    import re
+    from hy.lex.lexer import lexer
+    from hy.lex.parser import parser
+    from rply.errors import LexingError
+    tokens = []
+    err = None
+    for token in lexer.lex(src_string):
+        tokens.append(token)
+        try:
+            model, = parser.parse(
+                iter(tokens),
+                state=ParserState(src_string, filename=None))
+        except (LexingError, LexException) as e:
+            err = e
+        else:
+            return model, src_string[re.match(
+                r'.+\n' * (model.end_line - 1)
+                    + '.' * model.end_column,
+                src_string).end():]
+    if err:
+        raise err
+    raise ValueError("No form found")
+
+
 mangle_delim = 'X'
 
 

From 5bfc140b4d52969b60b4847159417c8c279a23cf Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 26 Feb 2019 14:04:24 -0500
Subject: [PATCH 113/223] Implement format strings

---
 NEWS.rst                       |   8 +++
 hy/compiler.py                 | 115 +++++++++++++++++++++++++++++++--
 hy/lex/lexer.py                |   2 +-
 hy/lex/parser.py               |  17 ++++-
 hy/models.py                   |   3 +-
 tests/compilers/test_ast.py    |  14 +++-
 tests/native_tests/language.hy |  66 +++++++++++++++++++
 7 files changed, 215 insertions(+), 10 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 7c66102..cccb8eb 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1,5 +1,13 @@
 .. default-role:: code
 
+Unreleased
+==============================
+
+New Features
+------------------------------
+* Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
+  are now supported, even on Pythons earlier than 3.6.
+
 0.16.0
 ==============================
 
diff --git a/hy/compiler.py b/hy/compiler.py
index 08e0c98..8bd2aae 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -12,14 +12,15 @@ from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
 from hy.errors import (HyCompileError, HyTypeError, HyLanguageError,
                        HySyntaxError, HyEvalError, HyInternalError)
 
-from hy.lex import mangle, unmangle
+from hy.lex import mangle, unmangle, hy_parse, parse_one_thing, LexException
 
 from hy._compat import (string_types, str_type, bytes_type, long_type, PY3,
-                        PY35, reraise)
+                        PY35, PY36, reraise)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
 import hy.core
 
+import re
 import pkgutil
 import traceback
 import importlib
@@ -31,6 +32,7 @@ import copy
 import __future__
 
 from collections import defaultdict
+from functools import reduce
 
 if PY3:
     import builtins
@@ -629,8 +631,11 @@ class HyASTCompiler(object):
         elif isinstance(form, HyKeyword):
             body = [HyString(form.name)]
 
-        elif isinstance(form, HyString) and form.brackets is not None:
-            body.extend([HyKeyword("brackets"), form.brackets])
+        elif isinstance(form, HyString):
+            if form.is_format:
+               body.extend([HyKeyword("is_format"), form.is_format])
+            if form.brackets is not None:
+               body.extend([HyKeyword("brackets"), form.brackets])
 
         ret = HyExpression([HySymbol(name)] + body).replace(form)
         return imports, ret, False
@@ -1798,10 +1803,112 @@ class HyASTCompiler(object):
 
     @builds_model(HyString, HyBytes)
     def compile_string(self, string):
+        if type(string) is HyString and string.is_format:
+            # This is a format string (a.k.a. an f-string).
+            return self._format_string(string, str_type(string))
         node = asty.Bytes if PY3 and type(string) is HyBytes else asty.Str
         f = bytes_type if type(string) is HyBytes else str_type
         return node(string, s=f(string))
 
+    def _format_string(self, string, rest, allow_recursion=True):
+        values = []
+        ret = Result()
+
+        while True:
+           # Look for the next replacement field, and get the
+           # plain text before it.
+           match = re.search(r'\{\{?|\}\}?', rest)
+           if match:
+              literal_chars = rest[: match.start()]
+              if match.group() == '}':
+                  raise self._syntax_error(string,
+                      "f-string: single '}' is not allowed")
+              if match.group() in ('{{', '}}'):
+                  # Doubled braces just add a single brace to the text.
+                  literal_chars += match.group()[0]
+              rest = rest[match.end() :]
+           else:
+              literal_chars = rest
+              rest = ""
+           if literal_chars:
+               values.append(asty.Str(string, s = literal_chars))
+           if not rest:
+               break
+           if match.group() != '{':
+               continue
+
+           # Look for the end of the replacement field, allowing
+           # one more level of matched braces, but no deeper, and only
+           # if we can recurse.
+           match = re.match(
+               r'(?: \{ [^{}]* \} | [^{}]+ )* \}'
+                   if allow_recursion
+                   else r'[^{}]* \}',
+               rest, re.VERBOSE)
+           if not match:
+              raise self._syntax_error(string, 'f-string: mismatched braces')
+           item = rest[: match.end() - 1]
+           rest = rest[match.end() :]
+
+           # Parse the first form.
+           try:
+               model, item = parse_one_thing(item)
+           except (ValueError, LexException) as e:
+               raise self._syntax_error(string, "f-string: " + str_type(e))
+
+           # Look for a conversion character.
+           item = item.lstrip()
+           conversion = None
+           if item.startswith('!'):
+               conversion = item[1]
+               item = item[2:].lstrip()
+
+           # Look for a format specifier.
+           format_spec = asty.Str(string, s = "")
+           if item.startswith(':'):
+               if allow_recursion:
+                   ret += self._format_string(string,
+                       item[1:],
+                       allow_recursion=False)
+                   format_spec = ret.force_expr
+               else:
+                   format_spec = asty.Str(string, s=item[1:])
+           elif item:
+               raise self._syntax_error(string,
+                   "f-string: trailing junk in field")
+
+           # Now, having finished compiling any recursively included
+           # forms, we can compile the first form that we parsed.
+           ret += self.compile(model)
+
+           if PY36:
+               values.append(asty.FormattedValue(
+                   string,
+                   conversion = -1 if conversion is None else ord(conversion),
+                   format_spec = format_spec,
+                   value = ret.force_expr))
+           else:
+               # Make an expression like:
+               #    "{!r:{}}".format(value, format_spec)
+               values.append(asty.Call(string,
+                   func = asty.Attribute(
+                       string,
+                       value = asty.Str(string, s =
+                           '{' +
+                           ('!' + conversion if conversion else '') +
+                           ':{}}'),
+                       attr = 'format', ctx = ast.Load()),
+                   args = [ret.force_expr, format_spec],
+                   keywords = [], starargs = None, kwargs = None))
+
+        return ret + (
+           asty.JoinedStr(string, values = values)
+           if PY36
+           else reduce(
+              lambda x, y:
+                  asty.BinOp(string, left = x, op = ast.Add(), right = y),
+              values))
+
     @builds_model(HyList, HySet)
     def compile_list(self, expression):
         elts, ret, _ = self._compile_collect(expression)
diff --git a/hy/lex/lexer.py b/hy/lex/lexer.py
index 14b7c88..f202d94 100755
--- a/hy/lex/lexer.py
+++ b/hy/lex/lexer.py
@@ -38,7 +38,7 @@ lg.add('HASHOTHER', r'#%s' % identifier)
 # A regexp which matches incomplete strings, used to support
 # multi-line strings in the interpreter
 partial_string = r'''(?x)
-    (?:u|r|ur|ru|b|br|rb)? # prefix
+    (?:u|r|ur|ru|b|br|rb|f|fr|rf)? # prefix
     "  # start string
     (?:
        | [^"\\]             # non-quote or backslash
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index 27180c5..6a1acfb 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -200,14 +200,22 @@ def t_empty_list(state, p):
 @pg.production("string : STRING")
 @set_boundaries
 def t_string(state, p):
+    s = p[0].value
+    # Detect and remove any "f" prefix.
+    is_format = False
+    if s.startswith('f') or s.startswith('rf'):
+        is_format = True
+        s = s.replace('f', '', 1)
     # Replace the single double quotes with triple double quotes to allow
     # embedded newlines.
     try:
-        s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""')
+        s = eval(s.replace('"', '"""', 1)[:-1] + '"""')
     except SyntaxError:
         raise LexException.from_lexer("Can't convert {} to a HyString".format(p[0].value),
                                       state, p[0])
-    return (HyString if isinstance(s, str_type) else HyBytes)(s)
+    return (HyString(s, is_format = is_format)
+        if isinstance(s, str_type)
+        else HyBytes(s))
 
 
 @pg.production("string : PARTIAL_STRING")
@@ -222,7 +230,10 @@ bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING')
 def t_bracket_string(state, p):
     m = bracket_string_re.match(p[0].value)
     delim, content = m.groups()
-    return HyString(content, brackets=delim)
+    return HyString(
+        content,
+        is_format = delim == 'f' or delim.startswith('f-'),
+        brackets = delim)
 
 
 @pg.production("identifier : IDENTIFIER")
diff --git a/hy/models.py b/hy/models.py
index ef51a26..458d615 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -94,8 +94,9 @@ class HyString(HyObject, str_type):
     scripts. It's either a ``str`` or a ``unicode``, depending on the
     Python version.
     """
-    def __new__(cls, s=None, brackets=None):
+    def __new__(cls, s=None, is_format=False, brackets=None):
         value = super(HyString, cls).__new__(cls, s)
+        value.is_format = bool(is_format)
         value.brackets = brackets
         return value
 
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 9311eef..9d004da 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -10,7 +10,7 @@ from hy.compiler import hy_compile, hy_eval
 from hy.errors import HyCompileError, HyLanguageError, HyError
 from hy.lex import hy_parse
 from hy.lex.exceptions import LexException, PrematureEndOfInput
-from hy._compat import PY3
+from hy._compat import PY3, PY36
 
 import ast
 import pytest
@@ -511,6 +511,18 @@ def test_ast_unicode_vs_bytes():
     assert s('b"\\xa0"') == (bytes([160]) if PY3 else chr(160))
 
 
+@pytest.mark.skipif(not PY36, reason='f-strings require Python 3.6+')
+def test_format_string():
+    assert can_compile('f"hello world"')
+    assert can_compile('f"hello {(+ 1 1)} world"')
+    assert can_compile('f"hello world {(+ 1 1)}"')
+    assert cant_compile('f"hello {(+ 1 1) world"')
+    assert cant_compile('f"hello (+ 1 1)} world"')
+    assert cant_compile('f"hello {(+ 1 1} world"')
+    assert can_compile(r'f"hello {\"n\"} world"')
+    assert can_compile(r'f"hello {\"\\n\"} world"')
+
+
 def test_ast_bracket_string():
     assert s(r'#[[empty delims]]') == 'empty delims'
     assert s(r'#[my delim[fizzle]my delim]') == 'fizzle'
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 65c629a..04936dd 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1217,6 +1217,72 @@
   (assert (none? (. '"squid" brackets))))
 
 
+(defn test-format-strings []
+  (assert (= f"hello world" "hello world"))
+  (assert (= f"hello {(+ 1 1)} world" "hello 2 world"))
+  (assert (= f"a{ (.upper (+ \"g\" \"k\")) }z" "aGKz"))
+
+  ; Referring to a variable
+  (setv p "xyzzy")
+  (assert (= f"h{p}j" "hxyzzyj"))
+
+  ; Including a statement and setting a variable
+  (assert (= f"a{(do (setv floop 4) (* floop 2))}z" "a8z"))
+  (assert (= floop 4))
+
+  ; Comments
+  (assert (= f"a{(+ 1
+     2 ; This is a comment.
+     3)}z" "a6z"))
+
+  ; Newlines in replacement fields
+  (assert (= f"ey {\"bee
+cee\"} dee" "ey bee\ncee dee"))
+
+  ; Conversion characters and format specifiers
+  (setv p:9 "other")
+  (setv !r "bar")
+  (defn u [s]
+    ; Add a "u" prefix for Python 2.
+    (if PY3
+      s
+      (.replace (.replace s "'" "u'" 1) "  " " " 1)))
+  (assert (= f"a{p !r}" (u "a'xyzzy'")))
+  (assert (= f"a{p :9}" "axyzzy    "))
+  (assert (= f"a{p:9}" "aother"))
+  (assert (= f"a{p !r :9}" (u "a'xyzzy'  ")))
+  (assert (= f"a{p !r:9}" (u "a'xyzzy'  ")))
+  (assert (= f"a{p:9 :9}" "aother    "))
+  (assert (= f"a{!r}" "abar"))
+  (assert (= f"a{!r !r}" (u "a'bar'")))
+
+  ; Fun with `r`
+  (assert (= f"hello {r\"\\n\"}" r"hello \n"))
+  (assert (= f"hello {r\"\n\"}" "hello \n"))
+    ; The `r` applies too late to avoid interpreting a backslash.
+
+  ; Braces escaped via doubling
+  (assert (= f"ab{{cde" "ab{cde"))
+  (assert (= f"ab{{cde}}}}fg{{{{{{" "ab{cde}}fg{{{"))
+  (assert (= f"ab{{{(+ 1 1)}}}" "ab{2}"))
+
+  ; Nested replacement fields
+  (assert (= f"{2 :{(+ 2 2)}}" "   2"))
+  (setv value 12.34  width 10  precision 4)
+  (assert (= f"result: {value :{width}.{precision}}" "result:      12.34"))
+
+  ; Nested replacement fields with ! and :
+  (defclass C [object]
+    (defn __format__ [self format-spec]
+      (+ "C[" format-spec "]")))
+  (assert (= f"{(C) :  {(str (+ 1 1)) !r :x<5}}" "C[  '2'xx]"))
+
+  ; Format bracket strings
+  (assert (= #[f[a{p !r :9}]f] (u "a'xyzzy'  ")))
+  (assert (= #[f-string[result: {value :{width}.{precision}}]f-string]
+    "result:      12.34")))
+
+
 (defn test-import-syntax []
   "NATIVE: test the import syntax."
 

From 83e56de0c53324e8d06f90fa475c6467166db8d4 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 27 Feb 2019 12:05:19 -0500
Subject: [PATCH 114/223] Document format strings

---
 docs/language/syntax.rst | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/docs/language/syntax.rst b/docs/language/syntax.rst
index 32ebec9..2414499 100644
--- a/docs/language/syntax.rst
+++ b/docs/language/syntax.rst
@@ -42,7 +42,8 @@ string literal called a "bracket string" similar to Lua's long brackets.
 Bracket strings have customizable delimiters, like the here-documents of other
 languages. A bracket string begins with ``#[FOO[`` and ends with ``]FOO]``,
 where ``FOO`` is any string not containing ``[`` or ``]``, including the empty
-string. For example::
+string. (If ``FOO`` is exactly ``f`` or begins with ``f-``, the bracket string
+is interpreted as a :ref:`format string <syntax-fstrings>`.) For example::
 
    => (print #[["That's very kind of yuo [sic]" Tom wrote back.]])
    "That's very kind of yuo [sic]" Tom wrote back.
@@ -69,6 +70,43 @@ of bytes. So when running under Python 3, Hy translates ``"foo"`` and
 
 Unlike Python, Hy only recognizes string prefixes (``r``, etc.) in lowercase.
 
+.. _syntax-fstrings:
+
+format strings
+--------------
+
+A format string (or "f-string", or "formatted string literal") is a string
+literal with embedded code, possibly accompanied by formatting commands. Hy
+f-strings work much like :ref:`Python f-strings <py:f-strings>` except that the
+embedded code is in Hy rather than Python, and they're supported on all
+versions of Python.
+
+::
+
+    => (print f"The sum is {(+ 1 1)}.")
+    The sum is 2.
+
+Since ``!`` and ``:`` are identifier characters in Hy, Hy decides where the
+code in a replacement field ends, and any conversion or format specifier
+begins, by parsing exactly one form. You can use ``do`` to combine several
+forms into one, as usual. Whitespace may be necessary to terminate the form::
+
+    => (setv foo "a")
+    => (print f"{foo:x<5}")
+    …
+    NameError: name 'hyx_fooXcolonXxXlessHthan_signX5' is not defined
+    => (print f"{foo :x<5}")
+    axxxx
+
+Unlike Python, whitespace is allowed between a conversion and a format
+specifier.
+
+Also unlike Python, comments and backslashes are allowed in replacement fields.
+Hy's lexer will still process the whole format string normally, like any other
+string, before any replacement fields are considered, so you may need to
+backslash your backslashes, and you can't comment out a closing brace or the
+string delimiter.
+
 .. _syntax-keywords:
 
 keywords

From 5d7b069ecb4c9540ec8e29f9230867f41bb9a18d Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Sat, 9 Mar 2019 07:43:41 +0000
Subject: [PATCH 115/223] Add collections indexes and slices tutorial

This change adds to the tutorial the hy way of accessing array.
---
 docs/tutorial.rst | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 70522e6..af5523f 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -5,7 +5,6 @@ Tutorial
 .. TODO
 ..
 ..  - How do I index into arrays or dictionaries?
-..  - How do I do array ranges?  e.g. x[5:] or y[2:10]
 ..  - Blow your mind with macros!
 ..  - Where's my banana???
 
@@ -342,6 +341,19 @@ The equivalent in Hy would be:
   (for [i (range 10)]
     (print (+ "'i' is now at " (str i))))
 
+Python's collections indexes and slices are implemented
+by the ``get`` and ``cut`` built-in:
+
+.. code-block:: clj
+
+  (setv array [0 1 2])
+  (get array 1)
+  (cut array -3 -1)
+
+which is equivalent to::
+
+  array[1]
+  array[-3:-1]
 
 You can also import and make use of various Python libraries.  For
 example:

From 9203fcaeb5ab06a294a7135a576de7811350db94 Mon Sep 17 00:00:00 2001
From: "Brandon T. Willard" <brandonwillard@gmail.com>
Date: Sat, 16 Mar 2019 23:19:49 -0500
Subject: [PATCH 116/223] Check arguments in with-decorator tag

Fixes #1757.
---
 hy/core/macros.hy | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 824a8e5..5e06751 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -240,6 +240,8 @@ name (i.e. `sys.argv[0]`).
 
 (deftag @ [expr]
   "with-decorator tag macro"
+  (if (empty? expr)
+      (macro-error expr "missing function argument"))
   (setv decorators (cut expr None -1)
         fndef (get expr -1))
   `(with-decorator ~@decorators ~fndef))

From 0fe7f42efcdde1010d16e93ca392059d000c0c01 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Mar 2019 18:28:39 -0400
Subject: [PATCH 117/223] Remove internal checks for Python 3.5

---
 hy/compiler.py    | 20 ++++++++++----------
 hy/core/shadow.hy |  6 +++---
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 8bd2aae..32610c5 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -15,7 +15,7 @@ from hy.errors import (HyCompileError, HyTypeError, HyLanguageError,
 from hy.lex import mangle, unmangle, hy_parse, parse_one_thing, LexException
 
 from hy._compat import (string_types, str_type, bytes_type, long_type, PY3,
-                        PY35, PY36, reraise)
+                        PY36, reraise)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
 import hy.core
@@ -106,7 +106,7 @@ def ast_str(x, piecewise=False):
 _special_form_compilers = {}
 _model_compilers = {}
 _decoratables = (ast.FunctionDef, ast.ClassDef)
-if PY35:
+if PY3:
     _decoratables += (ast.AsyncFunctionDef,)
 # _bad_roots are fake special operators, which are used internally
 # by other special forms (e.g., `except` in `try`) but can't be
@@ -279,7 +279,7 @@ class Result(object):
                 var.arg = new_name
             elif isinstance(var, ast.FunctionDef):
                 var.name = new_name
-            elif PY35 and isinstance(var, ast.AsyncFunctionDef):
+            elif PY3 and isinstance(var, ast.AsyncFunctionDef):
                 var.name = new_name
             else:
                 raise TypeError("Don't know how to rename a %s!" % (
@@ -475,7 +475,7 @@ class HyASTCompiler(object):
         exprs_iter = iter(exprs)
         for expr in exprs_iter:
 
-            if not PY35 and oldpy_unpack and is_unpack("iterable", expr):
+            if not PY3 and oldpy_unpack and is_unpack("iterable", expr):
                 if oldpy_starargs:
                     raise self._syntax_error(expr,
                         "Pythons < 3.5 allow only one `unpack-iterable` per call")
@@ -485,7 +485,7 @@ class HyASTCompiler(object):
 
             elif is_unpack("mapping", expr):
                 ret += self.compile(expr[1])
-                if PY35:
+                if PY3:
                     if dict_display:
                         compiled_exprs.append(None)
                         compiled_exprs.append(ret.force_expr)
@@ -912,7 +912,7 @@ class HyASTCompiler(object):
             ret += self.compile(arg)
         return ret + asty.Yield(expr, value=ret.force_expr)
 
-    @special([(PY3, "yield-from"), (PY35, "await")], [FORM])
+    @special([(PY3, "yield-from"), (PY3, "await")], [FORM])
     def compile_yield_from_or_await_expression(self, expr, root, arg):
         ret = Result() + self.compile(arg)
         node = asty.YieldFrom if root == "yield-from" else asty.Await
@@ -991,7 +991,7 @@ class HyASTCompiler(object):
         fn.stmts[-1].decorator_list = decs + fn.stmts[-1].decorator_list
         return ret + fn
 
-    @special(["with*", (PY35, "with/a*")],
+    @special(["with*", (PY3, "with/a*")],
              [brackets(FORM, maybe(FORM)), many(FORM)])
     def compile_with_expression(self, expr, root, args, body):
         thing, ctx = (None, args[0]) if args[1] is None else args
@@ -1348,11 +1348,11 @@ class HyASTCompiler(object):
              "|": ast.BitOr,
              "^": ast.BitXor,
              "&": ast.BitAnd}
-    if PY35:
+    if PY3:
         m_ops["@"] = ast.MatMult
 
     @special(["+", "*", "|"], [many(FORM)])
-    @special(["-", "/", "&", (PY35, "@")], [oneplus(FORM)])
+    @special(["-", "/", "&", (PY3, "@")], [oneplus(FORM)])
     @special(["**", "//", "<<", ">>"], [times(2, Inf, FORM)])
     @special(["%", "^"], [times(2, 2, FORM)])
     def compile_maths_expression(self, expr, root, args):
@@ -1478,7 +1478,7 @@ class HyASTCompiler(object):
 
     NASYM = some(lambda x: isinstance(x, HySymbol) and x not in (
         "&optional", "&rest", "&kwonly", "&kwargs"))
-    @special(["fn", "fn*", (PY35, "fn/a")], [
+    @special(["fn", "fn*", (PY3, "fn/a")], [
         # The starred version is for internal use (particularly, in the
         # definition of `defn`). It ensures that a FunctionDef is
         # produced rather than a Lambda.
diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy
index 9560d97..2135400 100644
--- a/hy/core/shadow.hy
+++ b/hy/core/shadow.hy
@@ -5,7 +5,7 @@
 ;;;; Hy shadow functions
 
 (import operator)
-(import [hy._compat [PY3 PY35]])
+(import [hy._compat [PY3]])
 
 (require [hy.core.bootstrap [*]])
 
@@ -60,7 +60,7 @@
   "Shadowed `%` operator takes `x` modulo `y`."
   (% x y))
 
-(if PY35
+(if PY3
     (defn @ [a1 &rest a-rest]
       "Shadowed `@` operator matrix multiples `a1` by each `a-rest`."
       (reduce operator.matmul a-rest a1)))
@@ -173,5 +173,5 @@
   'and 'or 'not
   'is 'is-not 'in 'not-in
   'get])
-(if (not PY35)
+(if (not PY3)
   (.remove EXPORTS '@))

From 85f203ba439e18606865699fd020bad737182e98 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Mar 2019 18:30:42 -0400
Subject: [PATCH 118/223] Remove checks in tests for Python 3.5

---
 tests/native_tests/language.hy    | 4 ++--
 tests/native_tests/mathematics.hy | 6 +++---
 tests/native_tests/operators.hy   | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 04936dd..01af5d5 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -11,7 +11,7 @@
         pytest)
 (import sys)
 
-(import [hy._compat [PY3 PY35 PY37]])
+(import [hy._compat [PY3 PY37]])
 
 (defn test-sys-argv []
   "NATIVE: test sys.argv"
@@ -1550,7 +1550,7 @@ cee\"} dee" "ey bee\ncee dee"))
 (defn test-disassemble []
   "NATIVE: Test the disassemble function"
   (assert (= (disassemble '(do (leaky) (leaky) (macros))) (cond
-    [PY35 "Module(
+    [PY3 "Module(
     body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))])"]
diff --git a/tests/native_tests/mathematics.hy b/tests/native_tests/mathematics.hy
index a4a61b0..aa4ad78 100644
--- a/tests/native_tests/mathematics.hy
+++ b/tests/native_tests/mathematics.hy
@@ -2,7 +2,7 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import [hy._compat [PY35]])
+(import [hy._compat [PY3]])
 
 (setv square (fn [x]
                (* x x)))
@@ -191,7 +191,7 @@
 
 (defn test-matmul []
   "NATIVE: test matrix multiplication"
-  (if PY35
+  (if PY3
     (assert (= (@ first-test-matrix second-test-matrix)
                product-of-test-matrices))
     ;; Python <= 3.4
@@ -205,6 +205,6 @@
   (setv matrix first-test-matrix
         matmul-attempt (try (@= matrix second-test-matrix)
                               (except [e [Exception]] e)))
-  (if PY35
+  (if PY3
     (assert (= product-of-test-matrices matrix))
     (assert (isinstance matmul-attempt NameError))))
diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy
index e08edbb..cb0851e 100644
--- a/tests/native_tests/operators.hy
+++ b/tests/native_tests/operators.hy
@@ -2,7 +2,7 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import pytest [hy._compat [PY35]])
+(import pytest [hy._compat [PY3]])
 
 (defmacro op-and-shadow-test [op &rest body]
   ; Creates two tests with the given `body`, one where all occurrences
@@ -102,7 +102,7 @@
   (forbid (f 1 2 3)))
 
 
-(when PY35 (op-and-shadow-test @
+(when PY3 (op-and-shadow-test @
   (defclass C [object] [
     __init__ (fn [self content] (setv self.content content))
     __matmul__ (fn [self other] (C (+ self.content other.content)))])

From efed0b6c2323b9615cfd260464517d4d1ce0c318 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Mar 2019 18:33:06 -0400
Subject: [PATCH 119/223] Move Python 3.5 tests to `py3_only_tests.hy`

---
 tests/native_tests/py35_only_tests.hy | 103 --------------------------
 tests/native_tests/py3_only_tests.hy  |  98 ++++++++++++++++++++++++
 2 files changed, 98 insertions(+), 103 deletions(-)
 delete mode 100644 tests/native_tests/py35_only_tests.hy

diff --git a/tests/native_tests/py35_only_tests.hy b/tests/native_tests/py35_only_tests.hy
deleted file mode 100644
index fe17443..0000000
--- a/tests/native_tests/py35_only_tests.hy
+++ /dev/null
@@ -1,103 +0,0 @@
-;; Copyright 2019 the authors.
-;; This file is part of Hy, which is free software licensed under the Expat
-;; license. See the LICENSE.
-
-;; Tests where the emitted code relies on Python ≥3.5.
-;; conftest.py skips this file when running on Python <3.5.
-
-(import [asyncio [get-event-loop sleep]])
-
-
-(defn test-unpacking-pep448-1star []
-  (setv l [1 2 3])
-  (setv p [4 5])
-  (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3]))
-  (assert (= (, "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
-  (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5}))
-  (defn f [&rest args] args)
-  (assert (= (f "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
-  (assert (= (+ #*l #*p) 15))
-  (assert (= (and #*l) 3)))
-
-
-(defn test-unpacking-pep448-2star []
-  (setv d1 {"a" 1 "b" 2})
-  (setv d2 {"c" 3 "d" 4})
-  (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
-  (defn fun [&optional a b c d e f] [a b c d e f])
-  (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))
-
-
-(defn run-coroutine [coro]
-  "Run a coroutine until its done in the default event loop."""
-  (.run_until_complete (get-event-loop) (coro)))
-
-
-(defn test-fn/a []
-  (assert (= (run-coroutine (fn/a [] (await (sleep 0)) [1 2 3]))
-             [1 2 3])))
-
-
-(defn test-defn/a []
-  (defn/a coro-test []
-    (await (sleep 0))
-    [1 2 3])
-  (assert (= (run-coroutine coro-test) [1 2 3])))
-
-
-(defn test-decorated-defn/a []
-  (defn decorator [func] (fn/a [] (/ (await (func)) 2)))
-
-  #@(decorator
-      (defn/a coro-test []
-        (await (sleep 0))
-        42))
-  (assert (= (run-coroutine coro-test) 21)))
-
-
-(defclass AsyncWithTest []
-  (defn --init-- [self val]
-    (setv self.val val)
-    None)
-
-  (defn/a --aenter-- [self]
-    self.val)
-
-  (defn/a --aexit-- [self tyle value traceback]
-    (setv self.val None)))
-
-
-(defn test-single-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t (AsyncWithTest 1)]
-        (assert (= t 1))))))
-
-(defn test-two-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)]
-        (assert (= t1 1))
-        (assert (= t2 2))))))
-
-(defn test-thrice-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)
-               t3 (AsyncWithTest 3)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3))))))
-
-(defn test-quince-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)
-               t3 (AsyncWithTest 3)
-               _ (AsyncWithTest 4)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3))))))
diff --git a/tests/native_tests/py3_only_tests.hy b/tests/native_tests/py3_only_tests.hy
index e9fd24a..4c79ad5 100644
--- a/tests/native_tests/py3_only_tests.hy
+++ b/tests/native_tests/py3_only_tests.hy
@@ -107,3 +107,101 @@
 
   (assert (= (. (MyClass) member-names)
              ["__module__" "__qualname__" "method1" "method2"])))
+
+
+(import [asyncio [get-event-loop sleep]])
+
+
+(defn test-unpacking-pep448-1star []
+  (setv l [1 2 3])
+  (setv p [4 5])
+  (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3]))
+  (assert (= (, "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
+  (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5}))
+  (defn f [&rest args] args)
+  (assert (= (f "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
+  (assert (= (+ #*l #*p) 15))
+  (assert (= (and #*l) 3)))
+
+
+(defn test-unpacking-pep448-2star []
+  (setv d1 {"a" 1 "b" 2})
+  (setv d2 {"c" 3 "d" 4})
+  (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
+  (defn fun [&optional a b c d e f] [a b c d e f])
+  (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))
+
+
+(defn run-coroutine [coro]
+  "Run a coroutine until its done in the default event loop."""
+  (.run_until_complete (get-event-loop) (coro)))
+
+
+(defn test-fn/a []
+  (assert (= (run-coroutine (fn/a [] (await (sleep 0)) [1 2 3]))
+             [1 2 3])))
+
+
+(defn test-defn/a []
+  (defn/a coro-test []
+    (await (sleep 0))
+    [1 2 3])
+  (assert (= (run-coroutine coro-test) [1 2 3])))
+
+
+(defn test-decorated-defn/a []
+  (defn decorator [func] (fn/a [] (/ (await (func)) 2)))
+
+  #@(decorator
+      (defn/a coro-test []
+        (await (sleep 0))
+        42))
+  (assert (= (run-coroutine coro-test) 21)))
+
+
+(defclass AsyncWithTest []
+  (defn --init-- [self val]
+    (setv self.val val)
+    None)
+
+  (defn/a --aenter-- [self]
+    self.val)
+
+  (defn/a --aexit-- [self tyle value traceback]
+    (setv self.val None)))
+
+
+(defn test-single-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t (AsyncWithTest 1)]
+        (assert (= t 1))))))
+
+(defn test-two-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)]
+        (assert (= t1 1))
+        (assert (= t2 2))))))
+
+(defn test-thrice-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))
+
+(defn test-quince-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)
+               _ (AsyncWithTest 4)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))

From ad97042b6b9ceaee494c61815ccedc13ac768d90 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Mar 2019 18:36:53 -0400
Subject: [PATCH 120/223] Don't test Python 3.4

---
 .travis.yml   | 1 -
 conftest.py   | 3 +--
 hy/_compat.py | 1 -
 setup.py      | 1 -
 4 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7a10e06..a7adf4b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,6 @@ sudo: false
 language: python
 python:
   - "2.7"
-  - "3.4"
   - "3.5"
   - "3.6"
   - pypy
diff --git a/conftest.py b/conftest.py
index 7d84e11..adc035b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -4,7 +4,7 @@ import importlib
 import py
 import pytest
 import hy
-from hy._compat import PY3, PY35, PY36
+from hy._compat import PY3, PY36
 
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
@@ -13,7 +13,6 @@ _fspath_pyimport = py.path.local.pyimport
 
 def pytest_ignore_collect(path, config):
     return (("py3_only" in path.basename and not PY3) or
-            ("py35_only" in path.basename and not PY35) or
             ("py36_only" in path.basename and not PY36) or None)
 
 
diff --git a/hy/_compat.py b/hy/_compat.py
index 92aa392..ddeb859 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -9,7 +9,6 @@ except ImportError:
 import sys, keyword, textwrap
 
 PY3 = sys.version_info[0] >= 3
-PY35 = sys.version_info >= (3, 5)
 PY36 = sys.version_info >= (3, 6)
 PY37 = sys.version_info >= (3, 7)
 
diff --git a/setup.py b/setup.py
index 54aba2b..67b2062 100755
--- a/setup.py
+++ b/setup.py
@@ -79,7 +79,6 @@ setup(
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",

From 30fc1425c1bb0d587f86092bc4dc7adbbd054acd Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 17 Mar 2019 18:40:37 -0400
Subject: [PATCH 121/223] Update docs and README

---
 NEWS.rst              |  4 ++++
 docs/language/api.rst | 10 ++--------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index cccb8eb..8fda6eb 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -3,6 +3,10 @@
 Unreleased
 ==============================
 
+Removals
+------------------------------
+* Python 3.4 is no longer supported.
+
 New Features
 ------------------------------
 * Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
diff --git a/docs/language/api.rst b/docs/language/api.rst
index c406679..93993d1 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -1678,20 +1678,14 @@ object (respectively) to provide positional or keywords arguments
     => (f #* [1 2] #** {"c" 3 "d" 4})
     [1, 2, 3, 4]
 
-With Python 3, you can unpack in an assignment list (:pep:`3132`).
+With Python 3, unpacking is allowed in more contexts, and you can unpack
+more than once in one expression (:pep:`3132`, :pep:`448`).
 
 .. code-block:: clj
 
     => (setv [a #* b c] [1 2 3 4 5])
     => [a b c]
     [1, [2, 3, 4], 5]
-
-With Python 3.5 or greater, unpacking is allowed in more contexts than just
-function calls, and you can unpack more than once in the same expression
-(:pep:`448`).
-
-.. code-block:: clj
-
     => [#* [1 2] #* [3 4]]
     [1, 2, 3, 4]
     => {#** {1 2} #** {3 4}}

From f87959f6ba56a9a94ceea2c9332763d5be091a28 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 18 Mar 2019 14:36:10 -0400
Subject: [PATCH 122/223] Use the Ubuntu Xenial image on Travis

We need it for Python 3.7.
---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a7adf4b..b630968 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,12 @@
 sudo: false
+dist: xenial
 language: python
 python:
   - "2.7"
   - "3.5"
   - "3.6"
-  - pypy
-  - pypy3
+  - pypy2.7-6.0
+  - pypy3.5-6.0
 install:
   - pip install -r requirements-travis.txt
   - pip install -e .

From cec940f36592cbce93c456f5ed0a855dc256ff4b Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 18 Mar 2019 14:36:22 -0400
Subject: [PATCH 123/223] Test Python 3.7 on Travis

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index b630968..02f2a51 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ python:
   - "2.7"
   - "3.5"
   - "3.6"
+  - "3.7"
   - pypy2.7-6.0
   - pypy3.5-6.0
 install:

From 920057c6213fd28c4a7ad25bb856f851df6f7ce2 Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Tue, 26 Mar 2019 01:48:13 +0000
Subject: [PATCH 124/223] Fix typo for HyList model name in the language
 internal doc

---
 docs/language/internals.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/language/internals.rst b/docs/language/internals.rst
index 5c3cb09..1cd32ad 100644
--- a/docs/language/internals.rst
+++ b/docs/language/internals.rst
@@ -64,9 +64,9 @@ objects in a macro, for instance.
 .. _hylist:
 
 HyList
-~~~~~~~~~~~~
+~~~~~~
 
-``hy.models.HyExpression`` is a :ref:`HySequence` for bracketed ``[]``
+``hy.models.HyList`` is a :ref:`HySequence` for bracketed ``[]``
 lists, which, when used as a top-level expression, translate to Python
 list literals in the compilation phase.
 

From da823d2cade03ab6085a70ce0a6634116fbf3547 Mon Sep 17 00:00:00 2001
From: "Andrew R. M" <nixy@nixy.moe>
Date: Sat, 6 Apr 2019 10:19:10 -0400
Subject: [PATCH 125/223] Fix a temporary-file crash

---
 NEWS.rst       | 5 +++++
 hy/importer.py | 8 +++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 8fda6eb..f1d7447 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -12,6 +12,11 @@ New Features
 * Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
   are now supported, even on Pythons earlier than 3.6.
 
+Bug Fixes
+------------------------------
+* Fixed a crash caused by errors creating temp files during bytecode
+  compilation
+
 0.16.0
 ==============================
 
diff --git a/hy/importer.py b/hy/importer.py
index 0e59498..6d31004 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -467,9 +467,10 @@ else:
         if cfile is None:
             cfile = cache_from_source(filename)
 
-        f = tempfile.NamedTemporaryFile('wb', dir=os.path.split(cfile)[0],
-                                        delete=False)
+        f = None
         try:
+            f = tempfile.NamedTemporaryFile('wb', dir=os.path.split(cfile)[0],
+                                            delete=False)
             f.write('\0\0\0\0')
             f.write(struct.pack('<I', timestamp))
             f.write(marshal.dumps(codeobject))
@@ -489,7 +490,8 @@ else:
             os.rename(f.name, cfile)
         except OSError:
             try:
-                os.unlink(f.name)
+                if f is not None:
+                    os.unlink(f.name)
             except OSError:
                 pass
 

From 9772b3ed864c82f1cc1ecf18324c3bba7c97ad6b Mon Sep 17 00:00:00 2001
From: "Andrew R. M" <nixy@nixy.moe>
Date: Sat, 6 Apr 2019 10:19:10 -0400
Subject: [PATCH 126/223] Update AUTHORS

---
 AUTHORS | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 0a6c72c..590223c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -89,4 +89,5 @@
 * Simon Gomizelj <simon@vodik.xyz>
 * Yigong Wang <wang@yigo.ng>
 * Oskar Kvist <oskar.kvist@gmail.com>
-* Brandon T. Willard <brandonwillard@gmail.com>
\ No newline at end of file
+* Brandon T. Willard <brandonwillard@gmail.com>
+* Andrew R. M. <nixy@nixy.moe>

From b0ed1039317c5c03f7fe9b1bd0dd535748c4856b Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Fri, 22 Mar 2019 02:06:36 +0000
Subject: [PATCH 127/223] add `list?` function to `hy.core`

`list?` will test if the argument is an instance of list.
---
 NEWS.rst                   |  1 +
 docs/language/core.rst     | 18 ++++++++++++++++++
 hy/core/language.hy        |  5 ++++-
 tests/native_tests/core.hy |  5 +++++
 4 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/NEWS.rst b/NEWS.rst
index f1d7447..540b399 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -11,6 +11,7 @@ New Features
 ------------------------------
 * Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
   are now supported, even on Pythons earlier than 3.6.
+* New list? function.
 
 Bug Fixes
 ------------------------------
diff --git a/docs/language/core.rst b/docs/language/core.rst
index 4237418..92d41d9 100644
--- a/docs/language/core.rst
+++ b/docs/language/core.rst
@@ -899,6 +899,24 @@ Returns the first logically-true value of ``(pred x)`` for any ``x`` in
    True
 
 
+.. _list?-fn:
+
+list?
+-----
+
+Usage: ``(list? x)``
+
+Returns ``True`` if *x* is a list.
+
+.. code-block:: hy
+
+   => (list? '(inc 41))
+   True
+
+   => (list? '42)
+   False
+
+
 .. _string?-fn:
 
 string?
diff --git a/hy/core/language.hy b/hy/core/language.hy
index ac046bb..3e1da65 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -203,6 +203,9 @@ Return series of accumulated sums (or other binary function results)."
   "Check if x is float."
   (isinstance x float))
 
+(defn list? [x]
+  (isinstance x list))
+
 (defn symbol? [s]
   "Check if `s` is a symbol."
   (instance? HySymbol s))
@@ -454,7 +457,7 @@ Even objects with the __name__ magic will work."
     disassemble drop drop-last drop-while empty? eval even? every? exec first
     filter flatten float? fraction gensym group-by identity inc input instance?
     integer integer? integer-char? interleave interpose islice iterable?
-    iterate iterator? juxt keyword keyword? last macroexpand
+    iterate iterator? juxt keyword keyword? last list? macroexpand
     macroexpand-1 mangle map merge-with multicombinations name neg? none? nth
     numeric? odd? partition permutations pos? product range read read-str
     remove repeat repeatedly rest reduce second some string string? symbol?
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index ab8eaf4..53efc3f 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -267,6 +267,11 @@ result['y in globals'] = 'y' in globals()")
   (assert-true (symbol? 'im-symbol))
   (assert-false (symbol? (name 'im-symbol))))
 
+(defn test-list? []
+  "NATIVE: testing the list? function"
+  (assert-false (list? "hello"))
+  (assert-true (list? [1 2 3])))
+
 (defn test-gensym []
   "NATIVE: testing the gensym function"
   (import [hy.models [HySymbol]])

From 1c7ca7ac1f71b8076e1195fa1d2952169898782b Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Tue, 9 Apr 2019 00:01:41 +0000
Subject: [PATCH 128/223] update contrib and macro to use the new list?
 function

---
 hy/contrib/hy_repr.hy | 2 +-
 hy/contrib/walk.hy    | 2 +-
 hy/core/macros.hy     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/hy/contrib/hy_repr.hy b/hy/contrib/hy_repr.hy
index 9fd59be..c99a45a 100644
--- a/hy/contrib/hy_repr.hy
+++ b/hy/contrib/hy_repr.hy
@@ -18,7 +18,7 @@
 
 (setv -registry {})
 (defn hy-repr-register [types f &optional placeholder]
-  (for [typ (if (instance? list types) types [types])]
+  (for [typ (if (list? types) types [types])]
     (setv (get -registry typ) (, f placeholder))))
 
 (setv -quoting False)
diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index a7a46e9..d30f29e 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -19,7 +19,7 @@
     (outer (HyExpression (map inner form)))]
    [(instance? HyDict form)
     (HyDict (outer (HyExpression (map inner form))))]
-   [(instance? list form)
+   [(list? form)
     ((type form) (outer (HyExpression (map inner form))))]
    [(coll? form)
     (walk inner outer (list form))]
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 5e06751..1021c76 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -213,7 +213,7 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
   (defn extract-o!-sym [arg]
     (cond [(and (symbol? arg) (.startswith arg "o!"))
            arg]
-          [(and (instance? list arg) (.startswith (first arg) "o!"))
+          [(and (list? arg) (.startswith (first arg) "o!"))
            (first arg)]))
   (setv os (list (filter identity (map extract-o!-sym args)))
         gs (lfor s os (HySymbol (+ "g!" (cut s 2)))))

From cb5c9ec4370c76393610eac85df93e166949d579 Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Tue, 9 Apr 2019 00:04:33 +0000
Subject: [PATCH 129/223] Add Tristan de Cacqueray to the AUTHORS

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 590223c..7e744e0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -91,3 +91,4 @@
 * Oskar Kvist <oskar.kvist@gmail.com>
 * Brandon T. Willard <brandonwillard@gmail.com>
 * Andrew R. M. <nixy@nixy.moe>
+* Tristan de Cacqueray <tdecacqu@redhat.com>

From 403442d6b1e12045086ffddaabc86ef32bc2d580 Mon Sep 17 00:00:00 2001
From: Evan Klitzke <evan@eklitzke.org>
Date: Thu, 28 Mar 2019 15:30:14 -0700
Subject: [PATCH 130/223] get_version is not needed in data_files

---
 setup.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/setup.py b/setup.py
index 67b2062..8ee8be6 100755
--- a/setup.py
+++ b/setup.py
@@ -58,9 +58,6 @@ setup(
         'hy.core': ['*.hy', '__pycache__/*'],
         'hy.extra': ['*.hy', '__pycache__/*'],
     },
-    data_files=[
-        ('get_version', ['get_version.py'])
-    ],
     author="Paul Tagliamonte",
     author_email="tag@pault.ag",
     long_description=long_description,

From d793cee90a36dc9504319040c12390cac1e07481 Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Tue, 9 Apr 2019 03:07:25 +0000
Subject: [PATCH 131/223] add `tuple?` function `hy.core`

`tuple?` will test if the argument is an instance of tuple.
---
 NEWS.rst                   |  1 +
 docs/language/core.rst     | 19 +++++++++++++++++++
 hy/core/language.hy        |  5 ++++-
 tests/native_tests/core.hy |  5 +++++
 4 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/NEWS.rst b/NEWS.rst
index 540b399..14842fb 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -12,6 +12,7 @@ New Features
 * Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
   are now supported, even on Pythons earlier than 3.6.
 * New list? function.
+* New tuple? function.
 
 Bug Fixes
 ------------------------------
diff --git a/docs/language/core.rst b/docs/language/core.rst
index 92d41d9..7e42691 100644
--- a/docs/language/core.rst
+++ b/docs/language/core.rst
@@ -951,6 +951,25 @@ Returns ``True`` if *x* is a symbol.
    => (symbol? '[a b c])
    False
 
+
+.. _tuple?-fn:
+
+tuple?
+------
+
+Usage: ``(tuple? x)``
+
+Returns ``True`` if *x* is a tuple.
+
+.. code-block:: hy
+
+   => (tuple? (, 42 44))
+   True
+
+   => (tuple? [42 44])
+   False
+
+
 .. _zero?-fn:
 
 zero?
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 3e1da65..93f2fa3 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -206,6 +206,9 @@ Return series of accumulated sums (or other binary function results)."
 (defn list? [x]
   (isinstance x list))
 
+(defn tuple? [x]
+  (isinstance x tuple))
+
 (defn symbol? [s]
   "Check if `s` is a symbol."
   (instance? HySymbol s))
@@ -461,4 +464,4 @@ Even objects with the __name__ magic will work."
     macroexpand-1 mangle map merge-with multicombinations name neg? none? nth
     numeric? odd? partition permutations pos? product range read read-str
     remove repeat repeatedly rest reduce second some string string? symbol?
-    take take-nth take-while unmangle xor tee zero? zip zip-longest])
+    take take-nth take-while tuple? unmangle xor tee zero? zip zip-longest])
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index 53efc3f..995ff53 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -272,6 +272,11 @@ result['y in globals'] = 'y' in globals()")
   (assert-false (list? "hello"))
   (assert-true (list? [1 2 3])))
 
+(defn test-tuple? []
+  "NATIVE: testing the tuple? function"
+  (assert-false (tuple? [4 5]))
+  (assert-true (tuple? (, 4 5))))
+
 (defn test-gensym []
   "NATIVE: testing the gensym function"
   (import [hy.models [HySymbol]])

From 9f519ed208a90d2c5365108847494e989528b2e4 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Apr 2019 14:52:44 -0400
Subject: [PATCH 132/223] Depend on astor master

We need it for Python 3.8.
---
 setup.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 8ee8be6..ef25ac5 100755
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,11 @@ class Install(install):
                         "." + filename[:-len(".hy")])
         install.run(self)
 
-install_requires = ['rply>=0.7.7', 'astor>=0.7.1', 'funcparserlib>=0.3.6', 'clint>=0.4']
+install_requires = [
+    'rply>=0.7.7',
+    'astor @ https://github.com/berkerpeksag/astor/archive/master.zip',
+    'funcparserlib>=0.3.6',
+    'clint>=0.4']
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 

From 0c7ada1e63092aa6031e39edd667eb50b94ec83f Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Apr 2019 15:36:42 -0400
Subject: [PATCH 133/223] Ignore SyntaxWarnings while testing

Python 3.8 introduces SyntaxWarnings for some things we test, like trying to call a string literal as if it were a function.
---
 setup.cfg | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.cfg b/setup.cfg
index e033c3b..3cce154 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -22,3 +22,4 @@ python_functions=test_* is_test_* hyx_test_* hyx_is_test_*
 filterwarnings =
     once::DeprecationWarning
     once::PendingDeprecationWarning
+    ignore::SyntaxWarning

From 8df0a41d7d01632f9ee733e277766986ae984228 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Apr 2019 15:38:03 -0400
Subject: [PATCH 134/223] Provide Module(..., type_ignores) for Python 3.8

---
 hy/_compat.py                  | 1 +
 hy/cmdline.py                  | 5 +++--
 hy/compiler.py                 | 2 +-
 tests/native_tests/language.hy | 7 ++++---
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index ddeb859..a2ab7a5 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -11,6 +11,7 @@ import sys, keyword, textwrap
 PY3 = sys.version_info[0] >= 3
 PY36 = sys.version_info >= (3, 6)
 PY37 = sys.version_info >= (3, 7)
+PY38 = sys.version_info >= (3, 8)
 
 # The value of UCS4 indicates whether Unicode strings are stored as UCS-4.
 # It is always true on Pythons >= 3.3, which use USC-4 on all systems.
diff --git a/hy/cmdline.py b/hy/cmdline.py
index f65379d..d38ec08 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -270,8 +270,9 @@ class HyREPL(code.InteractiveConsole, object):
             try:
                 # Mush the two AST chunks into a single module for
                 # conversion into Python.
-                new_ast = ast.Module(exec_ast.body +
-                                     [ast.Expr(eval_ast.body)])
+                new_ast = ast.Module(
+                    exec_ast.body + [ast.Expr(eval_ast.body)],
+                    type_ignores=[])
                 print(astor.to_source(new_ast))
             except Exception:
                 msg = 'Exception in AST callback:\n{}\n'.format(
diff --git a/hy/compiler.py b/hy/compiler.py
index 32610c5..750d8d2 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -2131,7 +2131,7 @@ def hy_compile(tree, module, root=ast.Module, get_expr=False,
                    key=lambda a: not (isinstance(a, ast.ImportFrom) and
                                       a.module == '__future__'))
 
-    ret = root(body=body)
+    ret = root(body=body, type_ignores=[])
 
     if get_expr:
         expr = ast.Expression(body=expr)
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 01af5d5..687376c 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -11,7 +11,7 @@
         pytest)
 (import sys)
 
-(import [hy._compat [PY3 PY37]])
+(import [hy._compat [PY3 PY37 PY38]])
 
 (defn test-sys-argv []
   "NATIVE: test sys.argv"
@@ -1550,10 +1550,11 @@ cee\"} dee" "ey bee\ncee dee"))
 (defn test-disassemble []
   "NATIVE: Test the disassemble function"
   (assert (= (disassemble '(do (leaky) (leaky) (macros))) (cond
-    [PY3 "Module(
+    [PY3 (.format "Module(
     body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
-        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))])"]
+        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))]{})"
+      (if PY38 ",\n    type_ignores=[]" ""))]
     [True "Module(
     body=[
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),

From f236ec8d9ac87d9ee8180765e7b37d9eccd83ea1 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Apr 2019 15:41:04 -0400
Subject: [PATCH 135/223] Test Python 3.8 on Travis

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 02f2a51..4234503 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,7 @@ python:
   - "3.5"
   - "3.6"
   - "3.7"
+  - 3.8-dev
   - pypy2.7-6.0
   - pypy3.5-6.0
 install:

From 63ba27b36d98a2b21f2f1511a12a53d98828961e Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Apr 2019 15:42:56 -0400
Subject: [PATCH 136/223] Update trove classifiers and NEWS

---
 NEWS.rst | 1 +
 setup.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 14842fb..f580a7f 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -9,6 +9,7 @@ Removals
 
 New Features
 ------------------------------
+* Python 3.8 is now supported.
 * Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
   are now supported, even on Pythons earlier than 3.6.
 * New list? function.
diff --git a/setup.py b/setup.py
index ef25ac5..71e84e4 100755
--- a/setup.py
+++ b/setup.py
@@ -83,6 +83,7 @@ setup(
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Topic :: Software Development :: Code Generators",
         "Topic :: Software Development :: Compilers",
         "Topic :: Software Development :: Libraries",

From 7b3ef423c1d267cb069bf746aaa2b196bb67aa2b Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 9 Apr 2019 16:12:25 -0400
Subject: [PATCH 137/223] Use html.escape instead of cgi.escape

cgi.escape is gone as of Python 3.8.
---
 docs/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index d537b60..682dbcf 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -2,7 +2,7 @@
 #
 # This file is execfile()d with the current directory set to its containing dir.
 
-import re, os, sys, time, cgi
+import re, os, sys, time, html
 sys.path.append(os.path.abspath(".."))
 
 extensions = ['sphinx.ext.intersphinx']
@@ -28,7 +28,7 @@ copyright = u'%s the authors' % time.strftime('%Y')
 version = ".".join(hy_version.split(".")[:-1])
 # The full version, including alpha/beta/rc tags.
 release = hy_version
-hy_descriptive_version = cgi.escape(hy_version)
+hy_descriptive_version = html.escape(hy_version)
 if "+" in hy_version:
     hy_descriptive_version += " <strong style='color: red;'>(unstable)</strong>"
 

From 6c74cf1f0748b0656c4f9eafb2b59a4053aa2e37 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 18 Apr 2019 14:28:19 -0400
Subject: [PATCH 138/223] Add `setx` for assignment expressions

---
 NEWS.rst                              |  1 +
 conftest.py                           |  5 +++--
 docs/language/api.rst                 | 15 ++++++++++++++
 hy/compiler.py                        | 21 +++++++++++--------
 tests/native_tests/py38_only_tests.hy | 29 +++++++++++++++++++++++++++
 5 files changed, 61 insertions(+), 10 deletions(-)
 create mode 100644 tests/native_tests/py38_only_tests.hy

diff --git a/NEWS.rst b/NEWS.rst
index f580a7f..18aa8ab 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -12,6 +12,7 @@ New Features
 * Python 3.8 is now supported.
 * Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
   are now supported, even on Pythons earlier than 3.6.
+* Added a special form `setx` to create Python 3.8 assignment expressions.
 * New list? function.
 * New tuple? function.
 
diff --git a/conftest.py b/conftest.py
index adc035b..9f65b48 100644
--- a/conftest.py
+++ b/conftest.py
@@ -4,7 +4,7 @@ import importlib
 import py
 import pytest
 import hy
-from hy._compat import PY3, PY36
+from hy._compat import PY3, PY36, PY38
 
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
@@ -13,7 +13,8 @@ _fspath_pyimport = py.path.local.pyimport
 
 def pytest_ignore_collect(path, config):
     return (("py3_only" in path.basename and not PY3) or
-            ("py36_only" in path.basename and not PY36) or None)
+            ("py36_only" in path.basename and not PY36) or
+            ("py38_only" in path.basename and not PY38) or None)
 
 
 def pyimport_patch_mismatch(self, **kwargs):
diff --git a/docs/language/api.rst b/docs/language/api.rst
index 93993d1..f37f0c4 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -412,6 +412,21 @@ They can be used to assign multiple variables at once:
     =>
 
 
+``setv`` always returns ``None``.
+
+
+setx
+-----
+
+Whereas ``setv`` creates an assignment statement, ``setx`` creates an assignment expression (see :pep:`572`). It requires Python 3.8 or later. Only one target–value pair is allowed, and the target must be a bare symbol, but the ``setx`` form returns the assigned value instead of ``None``.
+
+::
+
+    => (when (> (setx x (+ 1 2)) 0)
+    ...  (print x "is greater than 0"))
+    3 is greater than 0
+
+
 defclass
 --------
 
diff --git a/hy/compiler.py b/hy/compiler.py
index 750d8d2..9a20daf 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -15,7 +15,7 @@ from hy.errors import (HyCompileError, HyTypeError, HyLanguageError,
 from hy.lex import mangle, unmangle, hy_parse, parse_one_thing, LexException
 
 from hy._compat import (string_types, str_type, bytes_type, long_type, PY3,
-                        PY36, reraise)
+                        PY36, PY38, reraise)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
 import hy.core
@@ -1399,15 +1399,16 @@ class HyASTCompiler(object):
             expr, target=target, value=ret.force_expr, op=op())
 
     @special("setv", [many(FORM + FORM)])
+    @special((PY38, "setx"), [times(1, 1, SYM + FORM)])
     def compile_def_expression(self, expr, root, pairs):
         if not pairs:
             return asty.Name(expr, id='None', ctx=ast.Load())
         result = Result()
         for pair in pairs:
-            result += self._compile_assign(*pair)
+            result += self._compile_assign(root, *pair)
         return result
 
-    def _compile_assign(self, name, result):
+    def _compile_assign(self, root, name, result):
 
         str_name = "%s" % name
         if str_name in (["None"] + (["True", "False"] if PY3 else [])):
@@ -1427,14 +1428,18 @@ class HyASTCompiler(object):
                 and isinstance(name, HySymbol)
                 and '.' not in name):
             result.rename(name)
-            # Throw away .expr to ensure that (setv ...) returns None.
-            result.expr = None
+            if root != HySymbol("setx"):
+                # Throw away .expr to ensure that (setv ...) returns None.
+                result.expr = None
         else:
             st_name = self._storeize(name, ld_name)
-            result += asty.Assign(
+            node = (asty.NamedExpr
+                if root == HySymbol("setx")
+                else asty.Assign)
+            result += node(
                 name if hasattr(name, "start_line") else result,
-                targets=[st_name],
-                value=result.force_expr)
+                value=result.force_expr,
+                target=st_name, targets=[st_name])
 
         return result
 
diff --git a/tests/native_tests/py38_only_tests.hy b/tests/native_tests/py38_only_tests.hy
new file mode 100644
index 0000000..ca337a5
--- /dev/null
+++ b/tests/native_tests/py38_only_tests.hy
@@ -0,0 +1,29 @@
+;; Copyright 2019 the authors.
+;; This file is part of Hy, which is free software licensed under the Expat
+;; license. See the LICENSE.
+
+;; Tests where the emitted code relies on Python ≥3.8.
+;; conftest.py skips this file when running on Python <3.8.
+
+(import pytest)
+
+(defn test-setx []
+  (setx y (+ (setx x (+ "a" "b")) "c"))
+  (assert (= x "ab"))
+  (assert (= y "abc"))
+
+  (setv l [])
+  (for [x [1 2 3]]
+    (when (>= (setx y (+ x 8)) 10)
+      (.append l y)))
+  (assert (= l [10 11]))
+
+  (setv a ["apple" None "banana"])
+  (setv filtered (lfor
+    i (range (len a))
+    :if (is-not (setx v (get a i)) None)
+    v))
+  (assert (= filtered ["apple" "banana"]))
+  (assert (= v "banana"))
+  (with [(pytest.raises NameError)]
+    i))

From e77ce92635fa6cc442628ab8ac57f86152789e4f Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 21 Apr 2019 11:45:58 -0400
Subject: [PATCH 139/223] Simplify gensym format

---
 NEWS.rst                            |  3 +++
 docs/language/api.rst               |  4 ++--
 hy/core/language.hy                 |  4 ++--
 tests/native_tests/core.hy          |  6 +++---
 tests/native_tests/native_macros.hy | 18 +++++++++---------
 5 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 18aa8ab..f925d01 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -15,6 +15,9 @@ New Features
 * Added a special form `setx` to create Python 3.8 assignment expressions.
 * New list? function.
 * New tuple? function.
+* Gensyms now have a simpler format that's more concise when
+  mangled (e.g., `_hyx_XsemicolonXfooXvertical_lineX1235` is now
+  `_hyx_fooXUffffX1`).
 
 Bug Fixes
 ------------------------------
diff --git a/docs/language/api.rst b/docs/language/api.rst
index f37f0c4..2a690f6 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -968,10 +968,10 @@ written without accidental variable name clashes.
 .. code-block:: clj
 
    => (gensym)
-   u':G_1235'
+   HySymbol('_G\uffff1')
 
    => (gensym "x")
-   u':x_1236'
+   HySymbol('_x\uffff2')
 
 .. seealso::
 
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 93f2fa3..53af7fb 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -214,7 +214,7 @@ Return series of accumulated sums (or other binary function results)."
   (instance? HySymbol s))
 
 (import [threading [Lock]])
-(setv _gensym_counter 1234)
+(setv _gensym_counter 0)
 (setv _gensym_lock (Lock))
 
 (defn gensym [&optional [g "G"]]
@@ -224,7 +224,7 @@ Return series of accumulated sums (or other binary function results)."
   (global _gensym_lock)
   (.acquire _gensym_lock)
   (try (do (setv _gensym_counter (inc _gensym_counter))
-           (setv new_symbol (HySymbol (.format "_;{0}|{1}" g _gensym_counter))))
+           (setv new_symbol (HySymbol (.format "_{}\uffff{}" g _gensym_counter))))
        (finally (.release _gensym_lock)))
   new_symbol)
 
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index 995ff53..39a389c 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -282,12 +282,12 @@ result['y in globals'] = 'y' in globals()")
   (import [hy.models [HySymbol]])
   (setv s1 (gensym))
   (assert (isinstance s1 HySymbol))
-  (assert (= 0 (.find s1 "_;G|")))
+  (assert (= 0 (.find s1 "_G\uffff")))
   (setv s2 (gensym "xx"))
   (setv s3 (gensym "xx"))
-  (assert (= 0 (.find s2 "_;xx|")))
+  (assert (= 0 (.find s2 "_xx\uffff")))
   (assert (not (= s2 s3)))
-  (assert (not (= (str s2) (str s3)))))
+  (assert (not (= (string s2) (string s3)))))
 
 (defn test-identity []
   "NATIVE: testing the identity function"
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index d5c48c7..835939e 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -166,9 +166,9 @@
   (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
-  ;; and make sure there is something new that starts with _;G|
-  (assert (in (mangle "_;G|") s1))
-  (assert (in (mangle "_;G|") s2))
+  ;; and make sure there is something new that starts with _G\uffff
+  (assert (in (mangle "_G\uffff") s1))
+  (assert (in (mangle "_G\uffff") s2))
   ;; but make sure the two don't match each other
   (assert (not (= s1 s2))))
 
@@ -193,8 +193,8 @@
   (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
-  (assert (in (mangle "_;a|") s1))
-  (assert (in (mangle "_;a|") s2))
+  (assert (in (mangle "_a\uffff") s1))
+  (assert (in (mangle "_a\uffff") s2))
   (assert (not (= s1 s2))))
 
 (defn test-defmacro/g! []
@@ -217,8 +217,8 @@
   (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
-  (assert (in (mangle "_;res|") s1))
-  (assert (in (mangle "_;res|") s2))
+  (assert (in (mangle "_res\uffff") s1))
+  (assert (in (mangle "_res\uffff") s2))
   (assert (not (= s1 s2)))
 
   ;; defmacro/g! didn't like numbers initially because they
@@ -247,8 +247,8 @@
   (setv _ast2 (hy-compile (hy-parse macro1) __name__))
   (setv s1 (to_source _ast1))
   (setv s2 (to_source _ast2))
-  (assert (in (mangle "_;res|") s1))
-  (assert (in (mangle "_;res|") s2))
+  (assert (in (mangle "_res\uffff") s1))
+  (assert (in (mangle "_res\uffff") s2))
   (assert (not (= s1 s2)))
 
   ;; defmacro/g! didn't like numbers initially because they

From 71ea2b5f0e075d21bbb505d2aef85e6f09565744 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 19 May 2019 13:34:31 -0400
Subject: [PATCH 140/223] Depend on astor 0.8

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 71e84e4..b3d80bc 100755
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,7 @@ class Install(install):
 
 install_requires = [
     'rply>=0.7.7',
-    'astor @ https://github.com/berkerpeksag/astor/archive/master.zip',
+    'astor>=0.8',
     'funcparserlib>=0.3.6',
     'clint>=0.4']
 if os.name == 'nt':

From 03eff1374c1c099d3f06f424e5e4f0d86aaced4e Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 19 May 2019 13:34:55 -0400
Subject: [PATCH 141/223] Update NEWS for release

---
 NEWS.rst | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index f925d01..94db065 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1,8 +1,13 @@
 .. default-role:: code
 
-Unreleased
+0.17.0
 ==============================
 
+**Warning**: Hy 0.17.x will be the last Hy versions to support Python 2,
+and we expect 0.17.0 to be the only release in this line. By the time
+0.18.0 is released (in 2020, after CPython 2 has ceased being developed),
+Hy will only support Python 3.
+
 Removals
 ------------------------------
 * Python 3.4 is no longer supported.
@@ -13,16 +18,15 @@ New Features
 * Format strings with embedded Hy code (e.g., `f"The sum is {(+ x y)}"`)
   are now supported, even on Pythons earlier than 3.6.
 * Added a special form `setx` to create Python 3.8 assignment expressions.
-* New list? function.
-* New tuple? function.
+* Added new core functions `list?` and `tuple`.
 * Gensyms now have a simpler format that's more concise when
   mangled (e.g., `_hyx_XsemicolonXfooXvertical_lineX1235` is now
   `_hyx_fooXUffffX1`).
 
 Bug Fixes
 ------------------------------
-* Fixed a crash caused by errors creating temp files during bytecode
-  compilation
+* Fixed a crash caused by errors creating temporary files during
+  bytecode compilation.
 
 0.16.0
 ==============================

From bd7b8bf5e19e7a3dd99d1a98f4c405fadbe34732 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 10:19:15 -0400
Subject: [PATCH 142/223] Revert "get_version is not needed in data_files"

This reverts commit 403442d6b1e12045086ffddaabc86ef32bc2d580.

It turns out that without `get_version` in `data_files`, trying to install Hy from a setuptools-produced source distribution fails with an error that `get_version` can't be found.
---
 setup.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.py b/setup.py
index b3d80bc..71afe86 100755
--- a/setup.py
+++ b/setup.py
@@ -62,6 +62,9 @@ setup(
         'hy.core': ['*.hy', '__pycache__/*'],
         'hy.extra': ['*.hy', '__pycache__/*'],
     },
+    data_files=[
+        ('get_version', ['get_version.py'])
+    ],
     author="Paul Tagliamonte",
     author_email="tag@pault.ag",
     long_description=long_description,

From 0fd02bf52b7f4cb77d838e739f3e75f97f923bd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Krystian=20Rosi=C5=84ski?= <krystian.rosinski@gmail.com>
Date: Fri, 24 May 2019 20:54:09 +0200
Subject: [PATCH 143/223] Fix a typo in a tutorial example

---
 docs/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index af5523f..96f8f7f 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -530,7 +530,7 @@ In Hy:
 .. code-block:: clj
 
   (defclass Customer [models.Model]
-    [name (models.CharField :max-length 255})
+    [name (models.CharField :max-length 255)
      address (models.TextField)
      notes (models.TextField)])
 

From da855af5698bdbafc584d43b8f2680784ced6835 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 14:13:36 -0400
Subject: [PATCH 144/223] Remove Python 2 from trove classifiers

---
 setup.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/setup.py b/setup.py
index 71afe86..bd6718e 100755
--- a/setup.py
+++ b/setup.py
@@ -80,8 +80,6 @@ setup(
         "Operating System :: OS Independent",
         "Programming Language :: Lisp",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",

From 4a2e7e1bd077ee07cf7cd2b471c0f0a897e28af2 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 14:17:10 -0400
Subject: [PATCH 145/223] Integrate py3_only_tests into native_tests/language

---
 tests/native_tests/language.hy       | 201 ++++++++++++++++++++++++++
 tests/native_tests/py3_only_tests.hy | 207 ---------------------------
 2 files changed, 201 insertions(+), 207 deletions(-)
 delete mode 100644 tests/native_tests/py3_only_tests.hy

diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 687376c..1494d14 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1716,3 +1716,204 @@ macros()
   "Make sure relative imports work properly"
   (import [..resources [tlib]])
   (assert (= tlib.SECRET-MESSAGE "Hello World")))
+
+
+(defn test-exception-cause []
+  (try (raise ValueError :from NameError)
+       (except [e [ValueError]]
+         (assert (= (type (. e __cause__)) NameError)))))
+
+
+(defn test-kwonly []
+  "NATIVE: test keyword-only arguments"
+  ;; keyword-only with default works
+  (defn kwonly-foo-default-false [&kwonly [foo False]] foo)
+  (assert (= (kwonly-foo-default-false) False))
+  (assert (= (kwonly-foo-default-false :foo True) True))
+  ;; keyword-only without default ...
+  (defn kwonly-foo-no-default [&kwonly foo] foo)
+  (setv attempt-to-omit-default (try
+                                  (kwonly-foo-no-default)
+                                  (except [e [Exception]] e)))
+  ;; works
+  (assert (= (kwonly-foo-no-default :foo "quux") "quux"))
+  ;; raises TypeError with appropriate message if not supplied
+  (assert (isinstance attempt-to-omit-default TypeError))
+  (assert (in "missing 1 required keyword-only argument: 'foo'"
+              (. attempt-to-omit-default args [0])))
+  ;; keyword-only with other arg types works
+  (defn function-of-various-args [a b &rest args &kwonly foo &kwargs kwargs]
+    (, a b args foo kwargs))
+  (assert (= (function-of-various-args 1 2 3 4 :foo 5 :bar 6 :quux 7)
+             (, 1 2 (, 3 4)  5 {"bar" 6 "quux" 7}))))
+
+
+(defn test-extended-unpacking-1star-lvalues []
+  (setv [x #*y] [1 2 3 4])
+  (assert (= x 1))
+  (assert (= y [2 3 4]))
+  (setv [a #*b c] "ghijklmno")
+  (assert (= a "g"))
+  (assert (= b (list "hijklmn")))
+  (assert (= c "o")))
+
+
+(defn test-yield-from []
+  "NATIVE: testing yield from"
+  (defn yield-from-test []
+    (for [i (range 3)]
+      (yield i))
+    (yield-from [1 2 3]))
+  (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))
+
+
+(defn test-yield-from-exception-handling []
+  "NATIVE: Ensure exception handling in yield from works right"
+  (defn yield-from-subgenerator-test []
+    (yield 1)
+    (yield 2)
+    (yield 3)
+    (assert 0))
+  (defn yield-from-test []
+    (for [i (range 3)]
+      (yield i))
+    (try
+      (yield-from (yield-from-subgenerator-test))
+      (except [e AssertionError]
+        (yield 4))))
+  (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4])))
+
+(require [hy.contrib.walk [let]])
+
+(defn test-let-optional []
+  (let [a 1
+        b 6
+        d 2]
+       (defn foo [&kwonly [a a] b [c d]]
+         (, a b c))
+       (assert (= (foo :b "b")
+                  (, 1 "b" 2)))
+       (assert (= (foo :b 20 :a 10 :c 30)
+                  (, 10 20 30)))))
+
+(defn test-pep-3115 []
+  (defclass member-table [dict]
+    [--init-- (fn [self] (setv self.member-names []))
+
+     --setitem-- (fn [self key value]
+                   (if (not-in key self)
+                       (.append self.member-names key))
+                   (dict.--setitem-- self key value))])
+
+  (defclass OrderedClass [type]
+    [--prepare-- (classmethod (fn [metacls name bases] (member-table)))
+
+     --new-- (fn [cls name bases classdict]
+               (setv result (type.--new-- cls name bases (dict classdict)))
+               (setv result.member-names classdict.member-names)
+               result)])
+
+  (defclass MyClass [:metaclass OrderedClass]
+    [method1 (fn [self] (pass))
+     method2 (fn [self] (pass))])
+
+  (assert (= (. (MyClass) member-names)
+             ["__module__" "__qualname__" "method1" "method2"])))
+
+
+(import [asyncio [get-event-loop sleep]])
+
+
+(defn test-unpacking-pep448-1star []
+  (setv l [1 2 3])
+  (setv p [4 5])
+  (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3]))
+  (assert (= (, "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
+  (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5}))
+  (defn f [&rest args] args)
+  (assert (= (f "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
+  (assert (= (+ #*l #*p) 15))
+  (assert (= (and #*l) 3)))
+
+
+(defn test-unpacking-pep448-2star []
+  (setv d1 {"a" 1 "b" 2})
+  (setv d2 {"c" 3 "d" 4})
+  (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
+  (defn fun [&optional a b c d e f] [a b c d e f])
+  (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))
+
+
+(defn run-coroutine [coro]
+  "Run a coroutine until its done in the default event loop."""
+  (.run_until_complete (get-event-loop) (coro)))
+
+
+(defn test-fn/a []
+  (assert (= (run-coroutine (fn/a [] (await (sleep 0)) [1 2 3]))
+             [1 2 3])))
+
+
+(defn test-defn/a []
+  (defn/a coro-test []
+    (await (sleep 0))
+    [1 2 3])
+  (assert (= (run-coroutine coro-test) [1 2 3])))
+
+
+(defn test-decorated-defn/a []
+  (defn decorator [func] (fn/a [] (/ (await (func)) 2)))
+
+  #@(decorator
+      (defn/a coro-test []
+        (await (sleep 0))
+        42))
+  (assert (= (run-coroutine coro-test) 21)))
+
+
+(defclass AsyncWithTest []
+  (defn --init-- [self val]
+    (setv self.val val)
+    None)
+
+  (defn/a --aenter-- [self]
+    self.val)
+
+  (defn/a --aexit-- [self tyle value traceback]
+    (setv self.val None)))
+
+
+(defn test-single-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t (AsyncWithTest 1)]
+        (assert (= t 1))))))
+
+(defn test-two-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)]
+        (assert (= t1 1))
+        (assert (= t2 2))))))
+
+(defn test-thrice-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))
+
+(defn test-quince-with/a []
+  (run-coroutine
+    (fn/a []
+      (with/a [t1 (AsyncWithTest 1)
+               t2 (AsyncWithTest 2)
+               t3 (AsyncWithTest 3)
+               _ (AsyncWithTest 4)]
+        (assert (= t1 1))
+        (assert (= t2 2))
+        (assert (= t3 3))))))
diff --git a/tests/native_tests/py3_only_tests.hy b/tests/native_tests/py3_only_tests.hy
deleted file mode 100644
index 4c79ad5..0000000
--- a/tests/native_tests/py3_only_tests.hy
+++ /dev/null
@@ -1,207 +0,0 @@
-;; Copyright 2019 the authors.
-;; This file is part of Hy, which is free software licensed under the Expat
-;; license. See the LICENSE.
-
-;; Tests where the emitted code relies on Python 3.
-;; conftest.py skips this file when running on Python 2.
-
-
-(defn test-exception-cause []
-  (try (raise ValueError :from NameError)
-       (except [e [ValueError]]
-         (assert (= (type (. e __cause__)) NameError)))))
-
-
-(defn test-kwonly []
-  "NATIVE: test keyword-only arguments"
-  ;; keyword-only with default works
-  (defn kwonly-foo-default-false [&kwonly [foo False]] foo)
-  (assert (= (kwonly-foo-default-false) False))
-  (assert (= (kwonly-foo-default-false :foo True) True))
-  ;; keyword-only without default ...
-  (defn kwonly-foo-no-default [&kwonly foo] foo)
-  (setv attempt-to-omit-default (try
-                                  (kwonly-foo-no-default)
-                                  (except [e [Exception]] e)))
-  ;; works
-  (assert (= (kwonly-foo-no-default :foo "quux") "quux"))
-  ;; raises TypeError with appropriate message if not supplied
-  (assert (isinstance attempt-to-omit-default TypeError))
-  (assert (in "missing 1 required keyword-only argument: 'foo'"
-              (. attempt-to-omit-default args [0])))
-  ;; keyword-only with other arg types works
-  (defn function-of-various-args [a b &rest args &kwonly foo &kwargs kwargs]
-    (, a b args foo kwargs))
-  (assert (= (function-of-various-args 1 2 3 4 :foo 5 :bar 6 :quux 7)
-             (, 1 2 (, 3 4)  5 {"bar" 6 "quux" 7}))))
-
-
-(defn test-extended-unpacking-1star-lvalues []
-  (setv [x #*y] [1 2 3 4])
-  (assert (= x 1))
-  (assert (= y [2 3 4]))
-  (setv [a #*b c] "ghijklmno")
-  (assert (= a "g"))
-  (assert (= b (list "hijklmn")))
-  (assert (= c "o")))
-
-
-(defn test-yield-from []
-  "NATIVE: testing yield from"
-  (defn yield-from-test []
-    (for [i (range 3)]
-      (yield i))
-    (yield-from [1 2 3]))
-  (assert (= (list (yield-from-test)) [0 1 2 1 2 3])))
-
-
-(defn test-yield-from-exception-handling []
-  "NATIVE: Ensure exception handling in yield from works right"
-  (defn yield-from-subgenerator-test []
-    (yield 1)
-    (yield 2)
-    (yield 3)
-    (assert 0))
-  (defn yield-from-test []
-    (for [i (range 3)]
-      (yield i))
-    (try
-      (yield-from (yield-from-subgenerator-test))
-      (except [e AssertionError]
-        (yield 4))))
-  (assert (= (list (yield-from-test)) [0 1 2 1 2 3 4])))
-
-(require [hy.contrib.walk [let]])
-
-(defn test-let-optional []
-  (let [a 1
-        b 6
-        d 2]
-       (defn foo [&kwonly [a a] b [c d]]
-         (, a b c))
-       (assert (= (foo :b "b")
-                  (, 1 "b" 2)))
-       (assert (= (foo :b 20 :a 10 :c 30)
-                  (, 10 20 30)))))
-
-(defn test-pep-3115 []
-  (defclass member-table [dict]
-    [--init-- (fn [self] (setv self.member-names []))
-
-     --setitem-- (fn [self key value]
-                   (if (not-in key self)
-                       (.append self.member-names key))
-                   (dict.--setitem-- self key value))])
-
-  (defclass OrderedClass [type]
-    [--prepare-- (classmethod (fn [metacls name bases] (member-table)))
-
-     --new-- (fn [cls name bases classdict]
-               (setv result (type.--new-- cls name bases (dict classdict)))
-               (setv result.member-names classdict.member-names)
-               result)])
-
-  (defclass MyClass [:metaclass OrderedClass]
-    [method1 (fn [self] (pass))
-     method2 (fn [self] (pass))])
-
-  (assert (= (. (MyClass) member-names)
-             ["__module__" "__qualname__" "method1" "method2"])))
-
-
-(import [asyncio [get-event-loop sleep]])
-
-
-(defn test-unpacking-pep448-1star []
-  (setv l [1 2 3])
-  (setv p [4 5])
-  (assert (= ["a" #*l "b" #*p #*l] ["a" 1 2 3 "b" 4 5 1 2 3]))
-  (assert (= (, "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
-  (assert (= #{"a" #*l "b" #*p #*l} #{"a" "b" 1 2 3 4 5}))
-  (defn f [&rest args] args)
-  (assert (= (f "a" #*l "b" #*p #*l) (, "a" 1 2 3 "b" 4 5 1 2 3)))
-  (assert (= (+ #*l #*p) 15))
-  (assert (= (and #*l) 3)))
-
-
-(defn test-unpacking-pep448-2star []
-  (setv d1 {"a" 1 "b" 2})
-  (setv d2 {"c" 3 "d" 4})
-  (assert (= {1 "x" #**d1 #**d2 2 "y"} {"a" 1 "b" 2 "c" 3 "d" 4 1 "x" 2 "y"}))
-  (defn fun [&optional a b c d e f] [a b c d e f])
-  (assert (= (fun #**d1 :e "eee" #**d2) [1 2 3 4 "eee" None])))
-
-
-(defn run-coroutine [coro]
-  "Run a coroutine until its done in the default event loop."""
-  (.run_until_complete (get-event-loop) (coro)))
-
-
-(defn test-fn/a []
-  (assert (= (run-coroutine (fn/a [] (await (sleep 0)) [1 2 3]))
-             [1 2 3])))
-
-
-(defn test-defn/a []
-  (defn/a coro-test []
-    (await (sleep 0))
-    [1 2 3])
-  (assert (= (run-coroutine coro-test) [1 2 3])))
-
-
-(defn test-decorated-defn/a []
-  (defn decorator [func] (fn/a [] (/ (await (func)) 2)))
-
-  #@(decorator
-      (defn/a coro-test []
-        (await (sleep 0))
-        42))
-  (assert (= (run-coroutine coro-test) 21)))
-
-
-(defclass AsyncWithTest []
-  (defn --init-- [self val]
-    (setv self.val val)
-    None)
-
-  (defn/a --aenter-- [self]
-    self.val)
-
-  (defn/a --aexit-- [self tyle value traceback]
-    (setv self.val None)))
-
-
-(defn test-single-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t (AsyncWithTest 1)]
-        (assert (= t 1))))))
-
-(defn test-two-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)]
-        (assert (= t1 1))
-        (assert (= t2 2))))))
-
-(defn test-thrice-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)
-               t3 (AsyncWithTest 3)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3))))))
-
-(defn test-quince-with/a []
-  (run-coroutine
-    (fn/a []
-      (with/a [t1 (AsyncWithTest 1)
-               t2 (AsyncWithTest 2)
-               t3 (AsyncWithTest 3)
-               _ (AsyncWithTest 4)]
-        (assert (= t1 1))
-        (assert (= t2 2))
-        (assert (= t3 3))))))

From 5c7441b011768340e7a012fdba7cb7dd88446170 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 14:34:00 -0400
Subject: [PATCH 146/223] Remove non-native tests of Python 2

---
 tests/compilers/test_ast.py      | 102 ++++++++-----------------------
 tests/compilers/test_compiler.py |  12 +---
 tests/test_bin.py                |  15 ++---
 3 files changed, 33 insertions(+), 96 deletions(-)

diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 9d004da..70ac563 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -10,7 +10,7 @@ from hy.compiler import hy_compile, hy_eval
 from hy.errors import HyCompileError, HyLanguageError, HyError
 from hy.lex import hy_parse
 from hy.lex.exceptions import LexException, PrematureEndOfInput
-from hy._compat import PY3, PY36
+from hy._compat import PY36
 
 import ast
 import pytest
@@ -121,9 +121,8 @@ def test_ast_good_raise():
     can_compile("(raise e)")
 
 
-if PY3:
-    def test_ast_raise_from():
-        can_compile("(raise Exception :from NameError)")
+def test_ast_raise_from():
+    can_compile("(raise Exception :from NameError)")
 
 
 def test_ast_bad_raise():
@@ -205,16 +204,16 @@ def test_ast_bad_global():
     cant_compile("(global (foo))")
 
 
-if PY3:
-    def test_ast_good_nonlocal():
-        "Make sure AST can compile valid nonlocal"
-        can_compile("(nonlocal a)")
-        can_compile("(nonlocal foo bar)")
+def test_ast_good_nonlocal():
+    "Make sure AST can compile valid nonlocal"
+    can_compile("(nonlocal a)")
+    can_compile("(nonlocal foo bar)")
 
-    def test_ast_bad_nonlocal():
-        "Make sure AST can't compile invalid nonlocal"
-        cant_compile("(nonlocal)")
-        cant_compile("(nonlocal (foo))")
+
+def test_ast_bad_nonlocal():
+    "Make sure AST can't compile invalid nonlocal"
+    cant_compile("(nonlocal)")
+    cant_compile("(nonlocal (foo))")
 
 
 def test_ast_good_defclass():
@@ -226,7 +225,6 @@ def test_ast_good_defclass():
     can_compile("(defclass a [] None (print \"foo\"))")
 
 
-@pytest.mark.skipif(not PY3, reason="Python 3 supports class keywords")
 def test_ast_good_defclass_with_metaclass():
     "Make sure AST can compile valid defclass with keywords"
     can_compile("(defclass a [:metaclass b])")
@@ -299,21 +297,6 @@ import a dotted name."""
     cant_compile("(require [spam [foo.bar]])")
 
 
-def test_ast_no_pointless_imports():
-    def contains_import_from(code):
-        return any([isinstance(node, ast.ImportFrom)
-                   for node in can_compile(code).body])
-    # `reduce` is a builtin in Python 2, but not Python 3.
-    # The version of `map` that returns an iterator is a builtin in
-    # Python 3, but not Python 2.
-    if PY3:
-        assert contains_import_from("reduce")
-        assert not contains_import_from("map")
-    else:
-        assert not contains_import_from("reduce")
-        assert contains_import_from("map")
-
-
 def test_ast_good_get():
     "Make sure AST can compile valid get"
     can_compile("(get x y)")
@@ -454,29 +437,20 @@ def test_lambda_list_keywords_kwargs():
 
 
 def test_lambda_list_keywords_kwonly():
-    """Ensure we can compile functions with &kwonly if we're on Python
-    3, or fail with an informative message on Python 2."""
     kwonly_demo = "(fn [&kwonly a [b 2]] (print 1) (print a b))"
-    if PY3:
-        code = can_compile(kwonly_demo)
-        for i, kwonlyarg_name in enumerate(('a', 'b')):
-            assert kwonlyarg_name == code.body[0].args.kwonlyargs[i].arg
-        assert code.body[0].args.kw_defaults[0] is None
-        assert code.body[0].args.kw_defaults[1].n == 2
-    else:
-        exception = cant_compile(kwonly_demo)
-        assert isinstance(exception, HyLanguageError)
-        message = exception.args[0]
-        assert message == "&kwonly parameters require Python 3"
+    code = can_compile(kwonly_demo)
+    for i, kwonlyarg_name in enumerate(('a', 'b')):
+        assert kwonlyarg_name == code.body[0].args.kwonlyargs[i].arg
+    assert code.body[0].args.kw_defaults[0] is None
+    assert code.body[0].args.kw_defaults[1].n == 2
 
 
 def test_lambda_list_keywords_mixed():
     """ Ensure we can mix them up."""
     can_compile("(fn [x &rest xs &kwargs kw] (list x xs kw))")
     cant_compile("(fn [x &rest xs &fasfkey {bar \"baz\"}])")
-    if PY3:
-        can_compile("(fn [x &rest xs &kwonly kwoxs &kwargs kwxs]"
-                    "  (list x xs kwxs kwoxs))")
+    can_compile("(fn [x &rest xs &kwonly kwoxs &kwargs kwxs]"
+                "  (list x xs kwxs kwoxs))")
 
 
 def test_missing_keyword_argument_value():
@@ -504,11 +478,11 @@ def test_ast_unicode_strings():
 
 
 def test_ast_unicode_vs_bytes():
-    assert s('"hello"') == u"hello"
-    assert type(s('"hello"')) is (str if PY3 else unicode)  # noqa
-    assert s('b"hello"') == (eval('b"hello"') if PY3 else "hello")
-    assert type(s('b"hello"')) is (bytes if PY3 else str)
-    assert s('b"\\xa0"') == (bytes([160]) if PY3 else chr(160))
+    assert s('"hello"') == "hello"
+    assert type(s('"hello"')) is str
+    assert s('b"hello"') == b"hello"
+    assert type(s('b"hello"')) is bytes
+    assert s('b"\\xa0"') == bytes([160])
 
 
 @pytest.mark.skipif(not PY36, reason='f-strings require Python 3.6+')
@@ -528,7 +502,7 @@ def test_ast_bracket_string():
     assert s(r'#[my delim[fizzle]my delim]') == 'fizzle'
     assert s(r'#[[]]') == ''
     assert s(r'#[my delim[]my delim]') == ''
-    assert type(s('#[X[hello]X]')) is (str if PY3 else unicode)  # noqa
+    assert type(s('#[X[hello]X]')) is str
     assert s(r'#[X[raw\nstring]X]') == 'raw\\nstring'
     assert s(r'#[foozle[aa foozli bb ]foozle]') == 'aa foozli bb '
     assert s(r'#[([unbalanced](]') == 'unbalanced'
@@ -623,30 +597,6 @@ def test_lots_of_comment_lines():
     can_compile(1000 * ";\n")
 
 
-def test_exec_star():
-
-    code = can_compile('(exec* "print(5)")').body[0]
-    assert type(code) == (ast.Expr if PY3 else ast.Exec)
-    if not PY3:
-        assert code.body.s == "print(5)"
-        assert code.globals is None
-        assert code.locals is None
-
-    code = can_compile('(exec* "print(a)" {"a" 3})').body[0]
-    assert type(code) == (ast.Expr if PY3 else ast.Exec)
-    if not PY3:
-        assert code.body.s == "print(a)"
-        assert code.globals.keys[0].s == "a"
-        assert code.locals is None
-
-    code = can_compile('(exec* "print(a + b)" {"a" "x"} {"b" "y"})').body[0]
-    assert type(code) == (ast.Expr if PY3 else ast.Exec)
-    if not PY3:
-        assert code.body.s == "print(a + b)"
-        assert code.globals.keys[0].s == "a"
-        assert code.locals.keys[0].s == "b"
-
-
 def test_compiler_macro_tag_try():
     """Check that try forms within defmacro/deftag are compiled correctly"""
     # https://github.com/hylang/hy/issues/1350
@@ -654,13 +604,11 @@ def test_compiler_macro_tag_try():
     can_compile("(deftag foo [] (try None (except [] None)) `())")
 
 
-@pytest.mark.skipif(not PY3, reason="Python 3 required")
 def test_ast_good_yield_from():
     "Make sure AST can compile valid yield-from"
     can_compile("(yield-from [1 2])")
 
 
-@pytest.mark.skipif(not PY3, reason="Python 3 required")
 def test_ast_bad_yield_from():
     "Make sure AST can't compile invalid yield-from"
     cant_compile("(yield-from)")
diff --git a/tests/compilers/test_compiler.py b/tests/compilers/test_compiler.py
index bbb2970..9648c8f 100644
--- a/tests/compilers/test_compiler.py
+++ b/tests/compilers/test_compiler.py
@@ -6,7 +6,6 @@ import ast
 
 from hy import compiler
 from hy.models import HyExpression, HyList, HySymbol, HyInteger
-from hy._compat import PY3
 
 
 def make_expression(*args):
@@ -64,12 +63,5 @@ def test_compiler_yield_return():
     assert len(body) == 2
     assert isinstance(body[0], ast.Expr)
     assert isinstance(body[0].value, ast.Yield)
-
-    if PY3:
-        # From 3.3+, the final statement becomes a return value
-        assert isinstance(body[1], ast.Return)
-        assert isinstance(body[1].value, ast.BinOp)
-    else:
-        # In earlier versions, the expression is not returned
-        assert isinstance(body[1], ast.Expr)
-        assert isinstance(body[1].value, ast.BinOp)
+    assert isinstance(body[1], ast.Return)
+    assert isinstance(body[1].value, ast.BinOp)
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 06b3af9..1cfb3e5 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -10,7 +10,6 @@ import shlex
 import subprocess
 
 from hy.importer import cache_from_source
-from hy._compat import PY3
 
 import pytest
 
@@ -497,9 +496,8 @@ def test_bin_hy_tracebacks():
     os.environ['HY_DEBUG'] = ''
 
     def req_err(x):
-        assert x == '{}HyRequireError: No module named {}'.format(
-            'hy.errors.' if PY3 else '',
-            (repr if PY3 else str)('not_a_real_module'))
+        assert (x == 'hy.errors.HyRequireError: No module named '
+            "'not_a_real_module'")
 
     # Modeled after
     #   > python -c 'import not_a_real_module'
@@ -512,7 +510,7 @@ def test_bin_hy_tracebacks():
         del error_lines[-1]
     assert len(error_lines) <= 10
       # Rough check for the internal traceback filtering
-    req_err(error_lines[4 if PY3 else -1])
+    req_err(error_lines[4])
 
     _, error = run_cmd('hy -c "(require not-a-real-module)"', expect=1)
     error_lines = error.splitlines()
@@ -522,7 +520,7 @@ def test_bin_hy_tracebacks():
     output, error = run_cmd('hy -i "(require not-a-real-module)"')
     assert output.startswith('=> ')
     print(error.splitlines())
-    req_err(error.splitlines()[2 if PY3 else -3])
+    req_err(error.splitlines()[2])
 
     # Modeled after
     #   > python -c 'print("hi'
@@ -535,9 +533,8 @@ def test_bin_hy_tracebacks():
         r'Traceback \(most recent call last\):\n'
         r'  File "(?:<string>|string-[0-9a-f]+)", line 1\n'
         r'    \(print "\n'
-        r'           \^\n' +
-        r'{}PrematureEndOfInput: Partial string literal\n'.format(
-            r'hy\.lex\.exceptions\.' if PY3 else ''))
+        r'           \^\n'
+        r'hy.lex.exceptions.PrematureEndOfInput: Partial string literal\n')
     assert re.search(peoi_re, error)
 
     # Modeled after

From ea872c39835ce3f41685c031942d44fab3a58188 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 14:46:52 -0400
Subject: [PATCH 147/223] Remove native tests of Python 2

---
 tests/native_tests/comprehensions.hy  |  6 +--
 tests/native_tests/contrib/hy_repr.hy | 28 +++++--------
 tests/native_tests/core.hy            |  8 +---
 tests/native_tests/extra/reserved.hy  |  5 +--
 tests/native_tests/language.hy        | 60 ++++++++++-----------------
 tests/native_tests/mangling.hy        |  7 +---
 tests/native_tests/mathematics.hy     | 16 ++-----
 tests/native_tests/operators.hy       |  6 +--
 8 files changed, 43 insertions(+), 93 deletions(-)

diff --git a/tests/native_tests/comprehensions.hy b/tests/native_tests/comprehensions.hy
index 9d97cbd..206de29 100644
--- a/tests/native_tests/comprehensions.hy
+++ b/tests/native_tests/comprehensions.hy
@@ -1,7 +1,6 @@
 (import
   types
-  pytest
-  [hy._compat [PY3]])
+  pytest)
 
 
 (defn test-comprehension-types []
@@ -134,8 +133,7 @@
   ; An `lfor` that gets compiled to a real comprehension
   (setv x 0)
   (assert (= (lfor x [1 2 3] (inc x)) [2 3 4]))
-  (assert (= x (if PY3 0 3)))
-    ; Python 2 list comprehensions leak their variables.
+  (assert (= x 0))
 
   ; An `lfor` that gets compiled to a loop
   (setv x 0  l [])
diff --git a/tests/native_tests/contrib/hy_repr.hy b/tests/native_tests/contrib/hy_repr.hy
index 7867181..4a0a1ec 100644
--- a/tests/native_tests/contrib/hy_repr.hy
+++ b/tests/native_tests/contrib/hy_repr.hy
@@ -3,7 +3,7 @@
 ;; license. See the LICENSE.
 
 (import
-  [hy._compat [PY3 PY36 PY37]]
+  [hy._compat [PY36 PY37]]
   [math [isnan]]
   [hy.contrib.hy-repr [hy-repr hy-repr-register]])
 
@@ -79,10 +79,10 @@
   (assert (is (type (get orig 1)) float))
   (assert (is (type (get result 1)) HyFloat)))
 
-(when PY3 (defn test-dict-views []
+(defn test-dict-views []
   (assert (= (hy-repr (.keys {1 2})) "(dict-keys [1])"))
   (assert (= (hy-repr (.values {1 2})) "(dict-values [2])"))
-  (assert (= (hy-repr (.items {1 2})) "(dict-items [(, 1 2)])"))))
+  (assert (= (hy-repr (.items {1 2})) "(dict-items [(, 1 2)])")))
 
 (defn test-datetime []
   (import [datetime :as D])
@@ -91,9 +91,8 @@
     "(datetime.datetime 2009 1 15 15 27 5)"))
   (assert (= (hy-repr (D.datetime 2009 1 15 15 27 5 123))
     "(datetime.datetime 2009 1 15 15 27 5 123)"))
-  (when PY3
-    (assert (= (hy-repr (D.datetime 2009 1 15 15 27 5 123 :tzinfo D.timezone.utc))
-      "(datetime.datetime 2009 1 15 15 27 5 123 :tzinfo datetime.timezone.utc)")))
+  (assert (= (hy-repr (D.datetime 2009 1 15 15 27 5 123 :tzinfo D.timezone.utc))
+    "(datetime.datetime 2009 1 15 15 27 5 123 :tzinfo datetime.timezone.utc)"))
   (when PY36
     (assert (= (hy-repr (D.datetime 2009 1 15 15 27 5 :fold 1))
       "(datetime.datetime 2009 1 15 15 27 5 :fold 1)"))
@@ -114,17 +113,11 @@
 (defn test-collections []
   (import collections)
   (assert (= (hy-repr (collections.defaultdict :a 8))
-    (if PY3
-      "(defaultdict None {\"a\" 8})"
-      "(defaultdict None {b\"a\" 8})")))
+    "(defaultdict None {\"a\" 8})"))
   (assert (= (hy-repr (collections.defaultdict int :a 8))
-    (if PY3
-      "(defaultdict <class 'int'> {\"a\" 8})"
-      "(defaultdict <type 'int'> {b\"a\" 8})")))
+    "(defaultdict <class 'int'> {\"a\" 8})"))
   (assert (= (hy-repr (collections.Counter [15 15 15 15]))
-    (if PY3
-      "(Counter {15 4})"
-      "(Counter {15 (int 4)})")))
+    "(Counter {15 4})"))
   (setv C (collections.namedtuple "Fooey" ["cd" "a_b"]))
   (assert (= (hy-repr (C 11 12))
     "(Fooey :cd 11 :a_b 12)")))
@@ -155,9 +148,8 @@
   (setv mo (re.search "b+" "aaaabbbccc"))
   (assert (= (hy-repr mo)
     (.format
-      #[[<{} object; :span {} :match "bbb">]]
-      (if PY37 "re.Match" (+ (. (type mo) __module__) ".SRE_Match"))
-      (if PY3 "(, 4 7)" "(, (int 4) (int 7))")))))
+      #[[<{} object; :span (, 4 7) :match "bbb">]]
+      (if PY37 "re.Match" (+ (. (type mo) __module__) ".SRE_Match"))))))
 
 (defn test-hy-repr-custom []
 
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index 39a389c..787a390 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -2,8 +2,6 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import [hy._compat [PY3]])
-
 ;;;; some simple helpers
 
 (defn assert-true [x]
@@ -429,8 +427,7 @@ result['y in globals'] = 'y' in globals()")
   (assert-true (neg? -2))
   (assert-false (neg? 1))
   (assert-false (neg? 0))
-  (when PY3
-    (assert-requires-num neg?)))
+  (assert-requires-num neg?))
 
 (defn test-zero []
   "NATIVE: testing the zero? function"
@@ -519,8 +516,7 @@ result['y in globals'] = 'y' in globals()")
   (assert-true (pos? 2))
   (assert-false (pos? -1))
   (assert-false (pos? 0))
-  (when PY3
-    (assert-requires-num pos?)))
+  (assert-requires-num pos?))
 
 (defn test-remove []
   "NATIVE: testing the remove function"
diff --git a/tests/native_tests/extra/reserved.hy b/tests/native_tests/extra/reserved.hy
index ca67577..b1b9c7a 100644
--- a/tests/native_tests/extra/reserved.hy
+++ b/tests/native_tests/extra/reserved.hy
@@ -2,13 +2,12 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import [hy.extra.reserved [names]] [hy._compat [PY3]])
+(import [hy.extra.reserved [names]])
 
 (defn test-reserved []
   (assert (is (type (names)) frozenset))
   (assert (in "and" (names)))
-  (when PY3
-    (assert (in "False" (names))))
+  (assert (in "False" (names)))
   (assert (in "pass" (names)))
   (assert (in "class" (names)))
   (assert (in "defclass" (names)))
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 1494d14..63b7599 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -11,7 +11,7 @@
         pytest)
 (import sys)
 
-(import [hy._compat [PY3 PY37 PY38]])
+(import [hy._compat [PY38]])
 
 (defn test-sys-argv []
   "NATIVE: test sys.argv"
@@ -71,13 +71,12 @@
        (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
   (try (eval '(defn None [] (print "hello")))
        (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
-  (when PY3
-    (try (eval '(setv False 1))
-         (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
-    (try (eval '(setv True 0))
-         (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
-    (try (eval '(defn True [] (print "hello")))
-         (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))))
+  (try (eval '(setv False 1))
+       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+  (try (eval '(setv True 0))
+       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+  (try (eval '(defn True [] (print "hello")))
+       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))))
 
 
 (defn test-setv-pairs []
@@ -513,9 +512,7 @@
   (setv passed False)
   (try
    (raise)
-   ;; Python 2 raises IndexError here (due to the previous test)
-   ;; Python 3 raises RuntimeError
-   (except [[IndexError RuntimeError]]
+   (except [RuntimeError]
      (setv passed True)))
   (assert passed)
 
@@ -747,16 +744,11 @@
 (defn test-yield-with-return []
   "NATIVE: test yield with return"
   (defn gen [] (yield 3) "goodbye")
-  (if PY3
-    (do (setv gg (gen))
-        (assert (= 3 (next gg)))
-        (try (next gg)
-             (except [e StopIteration] (assert (hasattr e "value"))
-                                       (assert (= (getattr e "value") "goodbye")))))
-    (do (setv gg (gen))
-        (assert (= 3 (next gg)))
-        (try (next gg)
-             (except [e StopIteration] (assert (not (hasattr e "value"))))))))
+  (setv gg (gen))
+  (assert (= 3 (next gg)))
+  (try (next gg)
+       (except [e StopIteration] (assert (hasattr e "value"))
+                                 (assert (= (getattr e "value") "goodbye")))))
 
 
 (defn test-yield-in-try []
@@ -1242,19 +1234,14 @@ cee\"} dee" "ey bee\ncee dee"))
   ; Conversion characters and format specifiers
   (setv p:9 "other")
   (setv !r "bar")
-  (defn u [s]
-    ; Add a "u" prefix for Python 2.
-    (if PY3
-      s
-      (.replace (.replace s "'" "u'" 1) "  " " " 1)))
-  (assert (= f"a{p !r}" (u "a'xyzzy'")))
+  (assert (= f"a{p !r}" "a'xyzzy'"))
   (assert (= f"a{p :9}" "axyzzy    "))
   (assert (= f"a{p:9}" "aother"))
-  (assert (= f"a{p !r :9}" (u "a'xyzzy'  ")))
-  (assert (= f"a{p !r:9}" (u "a'xyzzy'  ")))
+  (assert (= f"a{p !r :9}" "a'xyzzy'  "))
+  (assert (= f"a{p !r:9}" "a'xyzzy'  "))
   (assert (= f"a{p:9 :9}" "aother    "))
   (assert (= f"a{!r}" "abar"))
-  (assert (= f"a{!r !r}" (u "a'bar'")))
+  (assert (= f"a{!r !r}" "a'bar'"))
 
   ; Fun with `r`
   (assert (= f"hello {r\"\\n\"}" r"hello \n"))
@@ -1278,7 +1265,7 @@ cee\"} dee" "ey bee\ncee dee"))
   (assert (= f"{(C) :  {(str (+ 1 1)) !r :x<5}}" "C[  '2'xx]"))
 
   ; Format bracket strings
-  (assert (= #[f[a{p !r :9}]f] (u "a'xyzzy'  ")))
+  (assert (= #[f[a{p !r :9}]f] "a'xyzzy'  "))
   (assert (= #[f-string[result: {value :{width}.{precision}}]f-string]
     "result:      12.34")))
 
@@ -1549,17 +1536,12 @@ cee\"} dee" "ey bee\ncee dee"))
 
 (defn test-disassemble []
   "NATIVE: Test the disassemble function"
-  (assert (= (disassemble '(do (leaky) (leaky) (macros))) (cond
-    [PY3 (.format "Module(
+  (assert (= (disassemble '(do (leaky) (leaky) (macros)))
+    (.format "Module(
     body=[Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[])),
         Expr(value=Call(func=Name(id='macros'), args=[], keywords=[]))]{})"
-      (if PY38 ",\n    type_ignores=[]" ""))]
-    [True "Module(
-    body=[
-        Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),
-        Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),
-        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[], starargs=None, kwargs=None))])"])))
+      (if PY38 ",\n    type_ignores=[]" ""))))
   (assert (= (disassemble '(do (leaky) (leaky) (macros)) True)
              "leaky()
 leaky()
diff --git a/tests/native_tests/mangling.hy b/tests/native_tests/mangling.hy
index 832d236..22c1942 100644
--- a/tests/native_tests/mangling.hy
+++ b/tests/native_tests/mangling.hy
@@ -3,9 +3,6 @@
 ;; license. See the LICENSE.
 
 
-(import [hy._compat [PY3]])
-
-
 (defn test-hyphen []
   (setv a-b 1)
   (assert (= a-b 1))
@@ -63,9 +60,7 @@
 (defn test-higher-unicode []
   (setv 😂 "emoji")
   (assert (= 😂 "emoji"))
-  (if PY3
-    (assert (= hyx_Xface_with_tears_of_joyX "emoji"))
-    (assert (= hyx_XU1f602X "emoji"))))
+  (assert (= hyx_Xface_with_tears_of_joyX "emoji")))
 
 
 (defn test-nameless-unicode []
diff --git a/tests/native_tests/mathematics.hy b/tests/native_tests/mathematics.hy
index aa4ad78..8ad8797 100644
--- a/tests/native_tests/mathematics.hy
+++ b/tests/native_tests/mathematics.hy
@@ -2,8 +2,6 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import [hy._compat [PY3]])
-
 (setv square (fn [x]
                (* x x)))
 
@@ -191,20 +189,12 @@
 
 (defn test-matmul []
   "NATIVE: test matrix multiplication"
-  (if PY3
-    (assert (= (@ first-test-matrix second-test-matrix)
-               product-of-test-matrices))
-    ;; Python <= 3.4
-    (do
-      (setv matmul-attempt (try (@ first-test-matrix second-test-matrix)
-                                (except [e [Exception]] e)))
-      (assert (isinstance matmul-attempt NameError)))))
+  (assert (= (@ first-test-matrix second-test-matrix)
+             product-of-test-matrices)))
 
 (defn test-augassign-matmul []
   "NATIVE: test augmented-assignment matrix multiplication"
   (setv matrix first-test-matrix
         matmul-attempt (try (@= matrix second-test-matrix)
                               (except [e [Exception]] e)))
-  (if PY3
-    (assert (= product-of-test-matrices matrix))
-    (assert (isinstance matmul-attempt NameError))))
+  (assert (= product-of-test-matrices matrix)))
diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy
index cb0851e..6f2ea4a 100644
--- a/tests/native_tests/operators.hy
+++ b/tests/native_tests/operators.hy
@@ -2,8 +2,6 @@
 ;; This file is part of Hy, which is free software licensed under the Expat
 ;; license. See the LICENSE.
 
-(import pytest [hy._compat [PY3]])
-
 (defmacro op-and-shadow-test [op &rest body]
   ; Creates two tests with the given `body`, one where all occurrences
   ; of the symbol `f` are syntactically replaced with `op` (a test of
@@ -102,14 +100,14 @@
   (forbid (f 1 2 3)))
 
 
-(when PY3 (op-and-shadow-test @
+(op-and-shadow-test @
   (defclass C [object] [
     __init__ (fn [self content] (setv self.content content))
     __matmul__ (fn [self other] (C (+ self.content other.content)))])
   (forbid (f))
   (assert (do (setv c (C "a")) (is (f c) c)))
   (assert (= (. (f (C "b") (C "c")) content) "bc"))
-  (assert (= (. (f (C "d") (C "e") (C "f")) content) "def"))))
+  (assert (= (. (f (C "d") (C "e") (C "f")) content) "def")))
 
 
 (op-and-shadow-test <<

From b130e3284e0edb362ddce194babdb0b08a47f222 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 14:47:08 -0400
Subject: [PATCH 148/223] Don't test Python 2

---
 .travis.yml | 2 --
 conftest.py | 5 ++---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 4234503..f7fe97b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,12 +2,10 @@ sudo: false
 dist: xenial
 language: python
 python:
-  - "2.7"
   - "3.5"
   - "3.6"
   - "3.7"
   - 3.8-dev
-  - pypy2.7-6.0
   - pypy3.5-6.0
 install:
   - pip install -r requirements-travis.txt
diff --git a/conftest.py b/conftest.py
index 9f65b48..7ded83e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -4,7 +4,7 @@ import importlib
 import py
 import pytest
 import hy
-from hy._compat import PY3, PY36, PY38
+from hy._compat import PY36, PY38
 
 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
 
@@ -12,8 +12,7 @@ _fspath_pyimport = py.path.local.pyimport
 
 
 def pytest_ignore_collect(path, config):
-    return (("py3_only" in path.basename and not PY3) or
-            ("py36_only" in path.basename and not PY36) or
+    return (("py36_only" in path.basename and not PY36) or
             ("py38_only" in path.basename and not PY38) or None)
 
 

From bba97ab2a6f8356d96e06e4eb9241f40aec6a53f Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 15:18:12 -0400
Subject: [PATCH 149/223] Remove hy._compat's type aliases

---
 hy/_compat.py         |  5 -----
 hy/compiler.py        | 19 +++++++++----------
 hy/completer.py       |  6 +++---
 hy/contrib/hy_repr.hy | 12 ++++--------
 hy/core/language.hy   |  5 ++---
 hy/lex/__init__.py    |  8 ++++----
 hy/lex/parser.py      |  3 +--
 hy/macros.py          |  4 ++--
 hy/models.py          | 32 ++++++++++++++------------------
 tests/test_models.py  | 15 +++++++--------
 10 files changed, 46 insertions(+), 63 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index a2ab7a5..0da0b09 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -17,11 +17,6 @@ PY38 = sys.version_info >= (3, 8)
 # It is always true on Pythons >= 3.3, which use USC-4 on all systems.
 UCS4 = sys.maxunicode == 0x10FFFF
 
-str_type     = str   if PY3 else unicode      # NOQA
-bytes_type   = bytes if PY3 else str          # NOQA
-long_type    = int   if PY3 else long         # NOQA
-string_types = str   if PY3 else basestring   # NOQA
-
 #
 # Inspired by the same-named `six` functions.
 #
diff --git a/hy/compiler.py b/hy/compiler.py
index 9a20daf..b1d6e2d 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -14,8 +14,7 @@ from hy.errors import (HyCompileError, HyTypeError, HyLanguageError,
 
 from hy.lex import mangle, unmangle, hy_parse, parse_one_thing, LexException
 
-from hy._compat import (string_types, str_type, bytes_type, long_type, PY3,
-                        PY36, PY38, reraise)
+from hy._compat import (PY3, PY36, PY38, reraise)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
 import hy.core
@@ -512,7 +511,7 @@ class HyASTCompiler(object):
                 compiled_value = self.compile(value)
                 ret += compiled_value
 
-                arg = str_type(expr)[1:]
+                arg = str(expr)[1:]
                 keywords.append(asty.keyword(
                     expr, arg=ast_str(arg), value=compiled_value.force_expr))
 
@@ -1358,7 +1357,7 @@ class HyASTCompiler(object):
     def compile_maths_expression(self, expr, root, args):
         if len(args) == 0:
             # Return the identity element for this operator.
-            return asty.Num(expr, n=long_type(
+            return asty.Num(expr, n=(
                 {"+": 0, "|": 0, "*": 1}[root]))
 
         if len(args) == 1:
@@ -1763,7 +1762,7 @@ class HyASTCompiler(object):
 
     @builds_model(HyInteger, HyFloat, HyComplex)
     def compile_numeric_literal(self, x):
-        f = {HyInteger: long_type,
+        f = {HyInteger: int,
              HyFloat: float,
              HyComplex: complex}[type(x)]
         return asty.Num(x, n=f(x))
@@ -1810,9 +1809,9 @@ class HyASTCompiler(object):
     def compile_string(self, string):
         if type(string) is HyString and string.is_format:
             # This is a format string (a.k.a. an f-string).
-            return self._format_string(string, str_type(string))
+            return self._format_string(string, str(string))
         node = asty.Bytes if PY3 and type(string) is HyBytes else asty.Str
-        f = bytes_type if type(string) is HyBytes else str_type
+        f = bytes if type(string) is HyBytes else str
         return node(string, s=f(string))
 
     def _format_string(self, string, rest, allow_recursion=True):
@@ -1859,7 +1858,7 @@ class HyASTCompiler(object):
            try:
                model, item = parse_one_thing(item)
            except (ValueError, LexException) as e:
-               raise self._syntax_error(string, "f-string: " + str_type(e))
+               raise self._syntax_error(string, "f-string: " + str(e))
 
            # Look for a conversion character.
            item = item.lstrip()
@@ -1933,7 +1932,7 @@ def get_compiler_module(module=None, compiler=None, calling_frame=False):
 
     module = getattr(compiler, 'module', None) or module
 
-    if isinstance(module, string_types):
+    if isinstance(module, str):
         if module.startswith('<') and module.endswith('>'):
             module = types.ModuleType(module)
         else:
@@ -2098,7 +2097,7 @@ def hy_compile(tree, module, root=ast.Module, get_expr=False,
     """
     module = get_compiler_module(module, compiler, False)
 
-    if isinstance(module, string_types):
+    if isinstance(module, str):
         if module.startswith('<') and module.endswith('>'):
             module = types.ModuleType(module)
         else:
diff --git a/hy/completer.py b/hy/completer.py
index 38cf8eb..0f65ae6 100644
--- a/hy/completer.py
+++ b/hy/completer.py
@@ -9,7 +9,7 @@ import sys
 
 import hy.macros
 import hy.compiler
-from hy._compat import builtins, string_types
+from hy._compat import builtins
 
 
 docomplete = True
@@ -78,7 +78,7 @@ class Completer(object):
         matches = []
         for p in self.path:
             for k in p.keys():
-                if isinstance(k, string_types):
+                if isinstance(k, str):
                     k = k.replace("_", "-")
                     if k.startswith(text):
                         matches.append(k)
@@ -89,7 +89,7 @@ class Completer(object):
         matches = []
         for p in self.tag_path:
             for k in p.keys():
-                if isinstance(k, string_types):
+                if isinstance(k, str):
                     if k.startswith(text):
                         matches.append("#{}".format(k))
         return matches
diff --git a/hy/contrib/hy_repr.hy b/hy/contrib/hy_repr.hy
index c99a45a..3baaebb 100644
--- a/hy/contrib/hy_repr.hy
+++ b/hy/contrib/hy_repr.hy
@@ -7,7 +7,7 @@
   re
   datetime
   collections
-  [hy._compat [PY3 PY36 str-type bytes-type long-type]]
+  [hy._compat [PY36]]
   [hy.models [HyObject HyExpression HySymbol HyKeyword HyInteger HyFloat HyComplex HyList HyDict HySet HyString HyBytes]])
 
 (try
@@ -84,10 +84,10 @@
     (+ "(" (-cat x) ")"))))
 
 (hy-repr-register [HySymbol HyKeyword] str)
-(hy-repr-register [str-type bytes-type] (fn [x]
+(hy-repr-register [str bytes] (fn [x]
   (setv r (.lstrip (-base-repr x) "ub"))
   (+
-    (if (instance? bytes-type x) "b" "")
+    (if (instance? bytes x) "b" "")
     (if (.startswith "\"" r)
       ; If Python's built-in repr produced a double-quoted string, use
       ; that.
@@ -96,10 +96,6 @@
       ; convert it.
       (+ "\"" (.replace (cut r 1 -1) "\"" "\\\"") "\"")))))
 (hy-repr-register bool str)
-(if (not PY3) (hy-repr-register int (fn [x]
-  (.format "(int {})" (-base-repr x)))))
-(if (not PY3) (hy-repr-register long_type (fn [x]
-  (.rstrip (-base-repr x) "L"))))
 (hy-repr-register float (fn [x]
   (if
     (isnan x)  "NaN"
@@ -131,7 +127,7 @@
     (-repr-time-innards x))))
 (defn -repr-time-innards [x]
   (.rstrip (+ " " (.join " " (filter identity [
-    (if x.microsecond (str-type x.microsecond))
+    (if x.microsecond (str x.microsecond))
     (if (not (none? x.tzinfo)) (+ ":tzinfo " (hy-repr x.tzinfo)))
     (if (and PY36 (!= x.fold 0)) (+ ":fold " (hy-repr x.fold)))])))))
 (defn -strftime-0 [x fmt]
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 53af7fb..687bd5c 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -11,7 +11,6 @@
 (import [fractions [Fraction :as fraction]])
 (import operator)  ; shadow not available yet
 (import sys)
-(import [hy._compat [long-type]]) ; long for python2, int for python3
 (import [hy.models [HySymbol HyKeyword]])
 (import [hy.lex [tokenize mangle unmangle read read-str]])
 (import [hy.lex.exceptions [LexException PrematureEndOfInput]])
@@ -254,11 +253,11 @@ Return series of accumulated sums (or other binary function results)."
 
 (defn integer [x]
   "Return Hy kind of integer for `x`."
-  (long-type x))
+  (int x))
 
 (defn integer? [x]
   "Check if `x` is an integer."
-  (isinstance x (, int long-type)))
+  (isinstance x int))
 
 (defn integer-char? [x]
   "Check if char `x` parses as an integer."
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index d133f5f..b29709c 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -8,7 +8,7 @@ import re
 import sys
 import unicodedata
 
-from hy._compat import str_type, isidentifier, UCS4
+from hy._compat import isidentifier, UCS4
 from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
 
@@ -116,7 +116,7 @@ def mangle(s):
 
     assert s
 
-    s = str_type(s)
+    s = str(s)
     s = s.replace("-", "_")
     s2 = s.lstrip('_')
     leading_underscores = '_' * (len(s) - len(s2))
@@ -147,7 +147,7 @@ def unmangle(s):
     form. This may not round-trip, because different Hy symbol names can
     mangle to the same Python identifier."""
 
-    s = str_type(s)
+    s = str(s)
 
     s2 = s.lstrip('_')
     leading_underscores = len(s) - len(s2)
@@ -203,4 +203,4 @@ def read(from_file=sys.stdin, eof=""):
 
 
 def read_str(input):
-    return read(StringIO(str_type(input)))
+    return read(StringIO(str(input)))
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index 6a1acfb..1d5ce24 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -9,7 +9,6 @@ from functools import wraps
 
 from rply import ParserGenerator
 
-from hy._compat import str_type
 from hy.models import (HyBytes, HyComplex, HyDict, HyExpression, HyFloat,
                        HyInteger, HyKeyword, HyList, HySet, HyString, HySymbol)
 from .lexer import lexer
@@ -214,7 +213,7 @@ def t_string(state, p):
         raise LexException.from_lexer("Can't convert {} to a HyString".format(p[0].value),
                                       state, p[0])
     return (HyString(s, is_format = is_format)
-        if isinstance(s, str_type)
+        if isinstance(s, str)
         else HyBytes(s))
 
 
diff --git a/hy/macros.py b/hy/macros.py
index e2cec31..97d0976 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -9,7 +9,7 @@ import traceback
 
 from contextlib import contextmanager
 
-from hy._compat import PY3, string_types, reraise, rename_function
+from hy._compat import PY3, reraise, rename_function
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
 from hy.errors import (HyLanguageError, HyMacroExpansionError, HyTypeError,
@@ -156,7 +156,7 @@ def require(source_module, target_module, assignments, prefix=""):
         parent_frame = inspect.stack()[1][0]
         target_namespace = parent_frame.f_globals
         target_module = target_namespace.get('__name__', None)
-    elif isinstance(target_module, string_types):
+    elif isinstance(target_module, str):
         target_module = importlib.import_module(target_module)
         target_namespace = target_module.__dict__
     elif inspect.ismodule(target_module):
diff --git a/hy/models.py b/hy/models.py
index 458d615..fbced02 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -6,7 +6,6 @@ from __future__ import unicode_literals
 from contextlib import contextmanager
 from math import isnan, isinf
 from hy import _initialize_env_var
-from hy._compat import PY3, str_type, bytes_type, long_type, string_types
 from hy.errors import HyWrapperError
 from fractions import Fraction
 from clint.textui import colored
@@ -88,7 +87,7 @@ def repr_indent(obj):
     return repr(obj).replace("\n", "\n  ")
 
 
-class HyString(HyObject, str_type):
+class HyString(HyObject, str):
     """
     Generic Hy String object. Helpful to store string literals from Hy
     scripts. It's either a ``str`` or a ``unicode``, depending on the
@@ -100,20 +99,20 @@ class HyString(HyObject, str_type):
         value.brackets = brackets
         return value
 
-_wrappers[str_type] = HyString
+_wrappers[str] = HyString
 
 
-class HyBytes(HyObject, bytes_type):
+class HyBytes(HyObject, bytes):
     """
     Generic Hy Bytes object. It's either a ``bytes`` or a ``str``, depending
     on the Python version.
     """
     pass
 
-_wrappers[bytes_type] = HyBytes
+_wrappers[bytes] = HyBytes
 
 
-class HySymbol(HyObject, str_type):
+class HySymbol(HyObject, str):
     """
     Hy Symbol. Basically a string.
     """
@@ -170,42 +169,39 @@ def strip_digit_separators(number):
     # Don't strip a _ or , if it's the first character, as _42 and
     # ,42 aren't valid numbers
     return (number[0] + number[1:].replace("_", "").replace(",", "")
-            if isinstance(number, string_types) and len(number) > 1
+            if isinstance(number, str) and len(number) > 1
             else number)
 
 
-class HyInteger(HyObject, long_type):
+class HyInteger(HyObject, int):
     """
     Internal representation of a Hy Integer. May raise a ValueError as if
-    int(foo) was called, given HyInteger(foo). On python 2.x long will
-    be used instead
+    int(foo) was called, given HyInteger(foo).
     """
 
     def __new__(cls, number, *args, **kwargs):
-        if isinstance(number, string_types):
+        if isinstance(number, str):
             number = strip_digit_separators(number)
             bases = {"0x": 16, "0o": 8, "0b": 2}
             for leader, base in bases.items():
                 if number.startswith(leader):
                     # We've got a string, known leader, set base.
-                    number = long_type(number, base=base)
+                    number = int(number, base=base)
                     break
             else:
                 # We've got a string, no known leader; base 10.
-                number = long_type(number, base=10)
+                number = int(number, base=10)
         else:
             # We've got a non-string; convert straight.
-            number = long_type(number)
+            number = int(number)
         return super(HyInteger, cls).__new__(cls, number)
 
 
 _wrappers[int] = HyInteger
-if not PY3:  # do not add long on python3
-    _wrappers[long_type] = HyInteger
 
 
 def check_inf_nan_cap(arg, value):
-    if isinstance(arg, string_types):
+    if isinstance(arg, str):
         if isinf(value) and "i" in arg.lower() and "Inf" not in arg:
             raise ValueError('Inf must be capitalized as "Inf"')
         if isnan(value) and "NaN" not in arg:
@@ -233,7 +229,7 @@ class HyComplex(HyObject, complex):
     """
 
     def __new__(cls, real, imag=0, *args, **kwargs):
-        if isinstance(real, string_types):
+        if isinstance(real, str):
             value = super(HyComplex, cls).__new__(
                 cls, strip_digit_separators(real)
             )
diff --git a/tests/test_models.py b/tests/test_models.py
index fd1b615..b9e6a90 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -5,14 +5,13 @@
 import copy
 import hy
 from clint.textui.colored import clean
-from hy._compat import long_type, str_type
 from hy.models import (wrap_value, replace_hy_obj, HyString, HyInteger, HyList,
                        HyDict, HySet, HyExpression, HyComplex, HyFloat, pretty)
 
 
-def test_wrap_long_type():
+def test_wrap_int():
     """ Test conversion of integers."""
-    wrapped = wrap_value(long_type(0))
+    wrapped = wrap_value(0)
     assert type(wrapped) == HyInteger
 
 
@@ -26,27 +25,27 @@ def test_wrap_tuple():
 
 def test_wrap_nested_expr():
     """ Test conversion of HyExpressions with embedded non-HyObjects."""
-    wrapped = wrap_value(HyExpression([long_type(0)]))
+    wrapped = wrap_value(HyExpression([0]))
     assert type(wrapped) == HyExpression
     assert type(wrapped[0]) == HyInteger
     assert wrapped == HyExpression([HyInteger(0)])
 
 
-def test_replace_long_type():
+def test_replace_int():
     """ Test replacing integers."""
-    replaced = replace_hy_obj(long_type(0), HyInteger(13))
+    replaced = replace_hy_obj(0, HyInteger(13))
     assert replaced == HyInteger(0)
 
 
 def test_replace_string_type():
     """Test replacing python string"""
-    replaced = replace_hy_obj(str_type("foo"), HyString("bar"))
+    replaced = replace_hy_obj("foo", HyString("bar"))
     assert replaced == HyString("foo")
 
 
 def test_replace_tuple():
     """ Test replacing tuples."""
-    replaced = replace_hy_obj((long_type(0), ), HyInteger(13))
+    replaced = replace_hy_obj((0, ), HyInteger(13))
     assert type(replaced) == HyList
     assert type(replaced[0]) == HyInteger
     assert replaced == HyList([HyInteger(0)])

From 2685b01a4b6e00901d6d5ea0877809abaef4be1d Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 15:18:56 -0400
Subject: [PATCH 150/223] Remove miscellaneous PY3 checks

---
 hy/cmdline.py     |  8 ++++----
 hy/core/shadow.hy | 13 ++++---------
 hy/macros.py      |  5 +----
 3 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/hy/cmdline.py b/hy/cmdline.py
index d38ec08..e6d3a71 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -35,7 +35,7 @@ from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
-from hy._compat import builtins, PY3, FileNotFoundError
+from hy._compat import builtins, FileNotFoundError
 
 
 sys.last_type = None
@@ -661,7 +661,7 @@ def hy2py_main():
     if options.with_source:
         # need special printing on Windows in case the
         # codepage doesn't support utf-8 characters
-        if PY3 and platform.system() == "Windows":
+        if platform.system() == "Windows":
             for h in hst:
                 try:
                     print(h)
@@ -676,7 +676,7 @@ def hy2py_main():
         _ast = hy_compile(hst, '__main__', filename=filename, source=source)
 
     if options.with_ast:
-        if PY3 and platform.system() == "Windows":
+        if platform.system() == "Windows":
             _print_for_windows(astor.dump_tree(_ast))
         else:
             print(astor.dump_tree(_ast))
@@ -684,7 +684,7 @@ def hy2py_main():
         print()
 
     if not options.without_python:
-        if PY3 and platform.system() == "Windows":
+        if platform.system() == "Windows":
             _print_for_windows(astor.code_gen.to_source(_ast))
         else:
             print(astor.code_gen.to_source(_ast))
diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy
index 2135400..8a18c60 100644
--- a/hy/core/shadow.hy
+++ b/hy/core/shadow.hy
@@ -5,12 +5,10 @@
 ;;;; Hy shadow functions
 
 (import operator)
-(import [hy._compat [PY3]])
 
 (require [hy.core.bootstrap [*]])
 
-(if PY3
-  (import [functools [reduce]]))
+(import [functools [reduce]])
 
 (defn + [&rest args]
   "Shadowed `+` operator adds `args`."
@@ -60,10 +58,9 @@
   "Shadowed `%` operator takes `x` modulo `y`."
   (% x y))
 
-(if PY3
-    (defn @ [a1 &rest a-rest]
-      "Shadowed `@` operator matrix multiples `a1` by each `a-rest`."
-      (reduce operator.matmul a-rest a1)))
+(defn @ [a1 &rest a-rest]
+  "Shadowed `@` operator matrix multiples `a1` by each `a-rest`."
+  (reduce operator.matmul a-rest a1)))
 
 (defn << [a1 a2 &rest a-rest]
   "Shadowed `<<` operator performs left-shift on `a1` by `a2`, ..., `a-rest`."
@@ -173,5 +170,3 @@
   'and 'or 'not
   'is 'is-not 'in 'not-in
   'get])
-(if (not PY3)
-  (.remove EXPORTS '@))
diff --git a/hy/macros.py b/hy/macros.py
index 97d0976..32658d6 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -9,7 +9,7 @@ import traceback
 
 from contextlib import contextmanager
 
-from hy._compat import PY3, reraise, rename_function
+from hy._compat import reraise, rename_function
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
 from hy.errors import (HyLanguageError, HyMacroExpansionError, HyTypeError,
@@ -74,9 +74,6 @@ def tag(name):
     def _(fn):
         _name = mangle('#{}'.format(name))
 
-        if not PY3:
-            _name = _name.encode('UTF-8')
-
         fn = rename_function(fn, _name)
 
         module = inspect.getmodule(fn)

From ecf0352d3702358d5ad3213280568e9292ce1f7b Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 16:12:00 -0400
Subject: [PATCH 151/223] Remove aliases: `builtins`, `FileNotFoundError`

---
 hy/_compat.py     | 9 ---------
 hy/cmdline.py     | 4 ++--
 hy/compiler.py    | 5 +----
 hy/completer.py   | 2 +-
 hy/core/shadow.hy | 2 +-
 tests/test_bin.py | 3 +--
 6 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 0da0b09..65be73c 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -2,10 +2,6 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
-try:
-    import __builtin__ as builtins
-except ImportError:
-    import builtins  # NOQA
 import sys, keyword, textwrap
 
 PY3 = sys.version_info[0] >= 3
@@ -97,8 +93,3 @@ def isidentifier(x):
     # https://bugs.python.org/issue33899
     tokens = [t for t in tokens if t[0] != T.NEWLINE]
     return len(tokens) == 2 and tokens[0][0] == T.NAME
-
-try:
-    FileNotFoundError = FileNotFoundError
-except NameError:
-    FileNotFoundError = IOError
diff --git a/hy/cmdline.py b/hy/cmdline.py
index e6d3a71..4c0c1fa 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -19,6 +19,7 @@ import time
 import linecache
 import hashlib
 import codeop
+import builtins
 
 import astor.code_gen
 
@@ -35,7 +36,6 @@ from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
-from hy._compat import builtins, FileNotFoundError
 
 
 sys.last_type = None
@@ -256,7 +256,7 @@ class HyREPL(code.InteractiveConsole, object):
                 module, f = '.'.join(parts[:-1]), parts[-1]
                 self.output_fn = getattr(importlib.import_module(module), f)
             else:
-                self.output_fn = __builtins__[mangle(output_fn)]
+                self.output_fn = getattr(builtins, mangle(output_fn))
 
         # Pre-mangle symbols for repl recent results: *1, *2, *3
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
diff --git a/hy/compiler.py b/hy/compiler.py
index b1d6e2d..3b0d700 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -28,15 +28,12 @@ import types
 import ast
 import sys
 import copy
+import builtins
 import __future__
 
 from collections import defaultdict
 from functools import reduce
 
-if PY3:
-    import builtins
-else:
-    import __builtin__ as builtins
 
 Inf = float('inf')
 
diff --git a/hy/completer.py b/hy/completer.py
index 0f65ae6..b1969c1 100644
--- a/hy/completer.py
+++ b/hy/completer.py
@@ -6,10 +6,10 @@ import contextlib
 import os
 import re
 import sys
+import builtins
 
 import hy.macros
 import hy.compiler
-from hy._compat import builtins
 
 
 docomplete = True
diff --git a/hy/core/shadow.hy b/hy/core/shadow.hy
index 8a18c60..74ab08f 100644
--- a/hy/core/shadow.hy
+++ b/hy/core/shadow.hy
@@ -60,7 +60,7 @@
 
 (defn @ [a1 &rest a-rest]
   "Shadowed `@` operator matrix multiples `a1` by each `a-rest`."
-  (reduce operator.matmul a-rest a1)))
+  (reduce operator.matmul a-rest a1))
 
 (defn << [a1 a2 &rest a-rest]
   "Shadowed `<<` operator performs left-shift on `a1` by `a2`, ..., `a-rest`."
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 1cfb3e5..297eade 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -8,13 +8,12 @@ import os
 import re
 import shlex
 import subprocess
+import builtins
 
 from hy.importer import cache_from_source
 
 import pytest
 
-from hy._compat import builtins
-
 
 hy_dir = os.environ.get('HY_DIR', '')
 

From c255f0d03c92a7d14ff379d6aee00a5360eeb143 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 15:43:39 -0400
Subject: [PATCH 152/223] Remove old hy._compat raising code

---
 hy/_compat.py | 46 ++++++++++------------------------------------
 1 file changed, 10 insertions(+), 36 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 65be73c..cbe0fad 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -2,7 +2,7 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
-import sys, keyword, textwrap
+import sys, keyword
 
 PY3 = sys.version_info[0] >= 3
 PY36 = sys.version_info >= (3, 6)
@@ -13,44 +13,18 @@ PY38 = sys.version_info >= (3, 8)
 # It is always true on Pythons >= 3.3, which use USC-4 on all systems.
 UCS4 = sys.maxunicode == 0x10FFFF
 
-#
-# Inspired by the same-named `six` functions.
-#
-if PY3:
-    raise_src = textwrap.dedent('''
-    def raise_from(value, from_value):
-        raise value from from_value
-    ''')
 
-    def reraise(exc_type, value, traceback=None):
-        try:
-            raise value.with_traceback(traceback)
-        finally:
-            traceback = None
+def reraise(exc_type, value, traceback=None):
+    try:
+        raise value.with_traceback(traceback)
+    finally:
+        traceback = None
 
-    code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize',
-                     'flags', 'code', 'consts', 'names', 'varnames',
-                     'filename', 'name', 'firstlineno', 'lnotab', 'freevars',
-                     'cellvars']
-else:
-    def raise_from(value, from_value=None):
-        raise value
-
-    raise_src = textwrap.dedent('''
-    def reraise(exc_type, value, traceback=None):
-        try:
-            raise exc_type, value, traceback
-        finally:
-            traceback = None
-    ''')
-
-    code_obj_args = ['argcount', 'nlocals', 'stacksize', 'flags', 'code',
-                     'consts', 'names', 'varnames', 'filename', 'name',
-                     'firstlineno', 'lnotab', 'freevars', 'cellvars']
-
-raise_code = compile(raise_src, __file__, 'exec')
-exec(raise_code)
 
+code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize',
+                 'flags', 'code', 'consts', 'names', 'varnames',
+                 'filename', 'name', 'firstlineno', 'lnotab', 'freevars',
+                 'cellvars']
 
 def rename_function(func, new_name):
     """Creates a copy of a function and [re]sets the name at the code-object

From 7991c59480fa2c7ca8a0ca691e134381861d641e Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 15:48:29 -0400
Subject: [PATCH 153/223] Remove handling of UCS-2

---
 hy/_compat.py      |  4 ----
 hy/lex/__init__.py | 17 ++---------------
 2 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index cbe0fad..518434c 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -9,10 +9,6 @@ PY36 = sys.version_info >= (3, 6)
 PY37 = sys.version_info >= (3, 7)
 PY38 = sys.version_info >= (3, 8)
 
-# The value of UCS4 indicates whether Unicode strings are stored as UCS-4.
-# It is always true on Pythons >= 3.3, which use USC-4 on all systems.
-UCS4 = sys.maxunicode == 0x10FFFF
-
 
 def reraise(exc_type, value, traceback=None):
     try:
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index b29709c..e2e5a26 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -8,7 +8,7 @@ import re
 import sys
 import unicodedata
 
-from hy._compat import isidentifier, UCS4
+from hy._compat import isidentifier
 from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
 
@@ -135,7 +135,7 @@ def mangle(s):
                else '{0}{1}{0}'.format(mangle_delim,
                    unicodedata.name(c, '').lower().replace('-', 'H').replace(' ', '_')
                    or 'U{}'.format(unicode_char_to_hex(c)))
-            for c in unicode_to_ucs4iter(s))
+            for c in s)
 
     s = leading_underscores + s
     assert isidentifier(s)
@@ -168,19 +168,6 @@ def unmangle(s):
     return '-' * leading_underscores + s
 
 
-def unicode_to_ucs4iter(ustr):
-    # Covert a unicode string to an iterable object,
-    # elements in the object are single USC-4 unicode characters
-    if UCS4:
-        return ustr
-    ucs4_list = list(ustr)
-    for i, u in enumerate(ucs4_list):
-        if 0xD7FF < ord(u) < 0xDC00:
-            ucs4_list[i] += ucs4_list[i + 1]
-            del ucs4_list[i + 1]
-    return ucs4_list
-
-
 def read(from_file=sys.stdin, eof=""):
     """Read from input and returns a tokenized string.
 

From e45cee575a4e6f9803f792fdab32c373571d4994 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 15:55:24 -0400
Subject: [PATCH 154/223] Move `rename_function` to hy.macros

---
 hy/_compat.py | 21 ---------------------
 hy/macros.py  | 23 ++++++++++++++++++++++-
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 518434c..1411521 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -17,27 +17,6 @@ def reraise(exc_type, value, traceback=None):
         traceback = None
 
 
-code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize',
-                 'flags', 'code', 'consts', 'names', 'varnames',
-                 'filename', 'name', 'firstlineno', 'lnotab', 'freevars',
-                 'cellvars']
-
-def rename_function(func, new_name):
-    """Creates a copy of a function and [re]sets the name at the code-object
-    level.
-    """
-    c = func.__code__
-    new_code = type(c)(*[getattr(c, 'co_{}'.format(a))
-                         if a != 'name' else str(new_name)
-                         for a in code_obj_args])
-
-    _fn = type(func)(new_code, func.__globals__, str(new_name),
-                     func.__defaults__, func.__closure__)
-    _fn.__dict__.update(func.__dict__)
-
-    return _fn
-
-
 def isidentifier(x):
     if x in ('True', 'False', 'None', 'print'):
         # `print` is special-cased here because Python 2's
diff --git a/hy/macros.py b/hy/macros.py
index 32658d6..3c91f11 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -9,7 +9,7 @@ import traceback
 
 from contextlib import contextmanager
 
-from hy._compat import reraise, rename_function
+from hy._compat import reraise
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
 from hy.errors import (HyLanguageError, HyMacroExpansionError, HyTypeError,
@@ -381,3 +381,24 @@ def tag_macroexpand(tag, tree, module):
         expr.module = inspect.getmodule(tag_macro)
 
     return replace_hy_obj(expr, tree)
+
+
+def rename_function(func, new_name):
+    """Creates a copy of a function and [re]sets the name at the code-object
+    level.
+    """
+    c = func.__code__
+    new_code = type(c)(*[getattr(c, 'co_{}'.format(a))
+                         if a != 'name' else str(new_name)
+                         for a in code_obj_args])
+
+    _fn = type(func)(new_code, func.__globals__, str(new_name),
+                     func.__defaults__, func.__closure__)
+    _fn.__dict__.update(func.__dict__)
+
+    return _fn
+
+code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize',
+                 'flags', 'code', 'consts', 'names', 'varnames',
+                 'filename', 'name', 'firstlineno', 'lnotab', 'freevars',
+                 'cellvars']

From d7da03be12e8c28afd7d27c96d75c6a61107345d Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 15:56:16 -0400
Subject: [PATCH 155/223] Simplify hy._compat.isidentifier

---
 hy/_compat.py | 23 ++---------------------
 1 file changed, 2 insertions(+), 21 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 1411521..57a67dd 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -18,27 +18,8 @@ def reraise(exc_type, value, traceback=None):
 
 
 def isidentifier(x):
-    if x in ('True', 'False', 'None', 'print'):
-        # `print` is special-cased here because Python 2's
-        # keyword.iskeyword will count it as a keyword, but we
-        # use the __future__ feature print_function, which makes
-        # it a non-keyword.
+    if x in ('True', 'False', 'None'):
         return True
     if keyword.iskeyword(x):
         return False
-    if PY3:
-        return x.isidentifier()
-    if x.rstrip() != x:
-        return False
-    import tokenize as T
-    from io import StringIO
-    try:
-        tokens = list(T.generate_tokens(StringIO(x).readline))
-    except (T.TokenError, IndentationError):
-        return False
-    # Some versions of Python 2.7 (including one that made it into
-    # Ubuntu 18.10) have a Python 3 backport that adds a NEWLINE
-    # token. Remove it if it's present.
-    # https://bugs.python.org/issue33899
-    tokens = [t for t in tokens if t[0] != T.NEWLINE]
-    return len(tokens) == 2 and tokens[0][0] == T.NAME
+    return x.isidentifier()

From 5dcb03b64dac3b94122da63a6a623c4b9da7b4a8 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 16:13:07 -0400
Subject: [PATCH 156/223] Move `isidentifier` to hy.lex

---
 hy/_compat.py      | 10 +---------
 hy/lex/__init__.py | 10 +++++++++-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 57a67dd..4b7d471 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -2,7 +2,7 @@
 # This file is part of Hy, which is free software licensed under the Expat
 # license. See the LICENSE.
 
-import sys, keyword
+import sys
 
 PY3 = sys.version_info[0] >= 3
 PY36 = sys.version_info >= (3, 6)
@@ -15,11 +15,3 @@ def reraise(exc_type, value, traceback=None):
         raise value.with_traceback(traceback)
     finally:
         traceback = None
-
-
-def isidentifier(x):
-    if x in ('True', 'False', 'None'):
-        return True
-    if keyword.iskeyword(x):
-        return False
-    return x.isidentifier()
diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py
index e2e5a26..c32742e 100644
--- a/hy/lex/__init__.py
+++ b/hy/lex/__init__.py
@@ -4,11 +4,11 @@
 
 from __future__ import unicode_literals
 
+import keyword
 import re
 import sys
 import unicodedata
 
-from hy._compat import isidentifier
 from hy.lex.exceptions import PrematureEndOfInput, LexException  # NOQA
 from hy.models import HyExpression, HySymbol
 
@@ -191,3 +191,11 @@ def read(from_file=sys.stdin, eof=""):
 
 def read_str(input):
     return read(StringIO(str(input)))
+
+
+def isidentifier(x):
+    if x in ('True', 'False', 'None'):
+        return True
+    if keyword.iskeyword(x):
+        return False
+    return x.isidentifier()

From 67def3359fdeff560181b536486a4887ec902664 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 16:14:07 -0400
Subject: [PATCH 157/223] Remove Python 2 support from hy.importer

---
 hy/importer.py                      | 408 ++--------------------------
 tests/importer/test_importer.py     |   6 +-
 tests/native_tests/native_macros.hy |   4 +-
 tests/test_bin.py                   |   2 +-
 4 files changed, 32 insertions(+), 388 deletions(-)

diff --git a/hy/importer.py b/hy/importer.py
index 6d31004..f54803d 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -19,33 +19,6 @@ from contextlib import contextmanager
 
 from hy.compiler import hy_compile, hy_ast_compile_flags
 from hy.lex import hy_parse
-from hy._compat import PY3
-
-
-def cache_from_source(source_path):
-    """Get the cached bytecode file name for a given source file name.
-
-    This function's name is set to mirror Python 3.x's
-    `importlib.util.cache_from_source`, which is also used when available.
-
-    Parameters
-    ----------
-    source_path : str
-        Path of the source file
-
-    Returns
-    -------
-    out : str
-        Path of the corresponding bytecode file that may--or may
-        not--actually exist.
-    """
-    if PY3:
-        return importlib.util.cache_from_source(source_path)
-    else:
-        # If source_path has a file extension, replace it with ".pyc".
-        # Otherwise, just append ".pyc".
-        d, f = os.path.split(source_path)
-        return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
 
 
 @contextmanager
@@ -135,370 +108,41 @@ def _get_code_from_file(run_name, fname=None,
                 source = f.read().decode('utf-8')
             code = compile(source, fname, 'exec')
 
-    return (code, fname) if PY3 else code
+    return (code, fname)
 
 
-if PY3:
-    importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy')
-    _py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code
+importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy')
+_py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code
 
-    def _could_be_hy_src(filename):
-        return (os.path.isfile(filename) and
-            (filename.endswith('.hy') or
-             not any(filename.endswith(ext)
-                     for ext in importlib.machinery.SOURCE_SUFFIXES[1:])))
+def _could_be_hy_src(filename):
+    return (os.path.isfile(filename) and
+        (filename.endswith('.hy') or
+         not any(filename.endswith(ext)
+                 for ext in importlib.machinery.SOURCE_SUFFIXES[1:])))
 
-    def _hy_source_to_code(self, data, path, _optimize=-1):
-        if _could_be_hy_src(path):
-            source = data.decode("utf-8")
-            hy_tree = hy_parse(source, filename=path)
-            with loader_module_obj(self) as module:
-                data = hy_compile(hy_tree, module)
+def _hy_source_to_code(self, data, path, _optimize=-1):
+    if _could_be_hy_src(path):
+        source = data.decode("utf-8")
+        hy_tree = hy_parse(source, filename=path)
+        with loader_module_obj(self) as module:
+            data = hy_compile(hy_tree, module)
 
-        return _py_source_to_code(self, data, path, _optimize=_optimize)
+    return _py_source_to_code(self, data, path, _optimize=_optimize)
 
-    importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code
+importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code
 
-    #  This is actually needed; otherwise, pre-created finders assigned to the
-    #  current dir (i.e. `''`) in `sys.path` will not catch absolute imports of
-    #  directory-local modules!
-    sys.path_importer_cache.clear()
+#  This is actually needed; otherwise, pre-created finders assigned to the
+#  current dir (i.e. `''`) in `sys.path` will not catch absolute imports of
+#  directory-local modules!
+sys.path_importer_cache.clear()
 
-    # Do this one just in case?
-    importlib.invalidate_caches()
-
-    # XXX: These and the 2.7 counterparts below aren't truly cross-compliant.
-    # They're useful for testing, though.
-    HyImporter = importlib.machinery.FileFinder
-    HyLoader = importlib.machinery.SourceFileLoader
-
-else:
-    import imp
-    import py_compile
-    import marshal
-    import struct
-    import traceback
-
-    from pkgutil import ImpImporter, ImpLoader
-
-    def _could_be_hy_src(filename):
-        return (filename.endswith('.hy') or
-                (os.path.isfile(filename) and
-                 not any(filename.endswith(s[0]) for s in imp.get_suffixes())))
-
-    class HyLoader(ImpLoader, object):
-        def __init__(self, fullname, filename, fileobj=None, etc=None):
-            """This constructor is designed for some compatibility with
-            SourceFileLoader."""
-            if etc is None and filename is not None:
-                if _could_be_hy_src(filename):
-                    etc = ('.hy', 'U', imp.PY_SOURCE)
-                    if fileobj is None:
-                        fileobj = io.open(filename, 'rU', encoding='utf-8')
-
-            super(HyLoader, self).__init__(fullname, fileobj, filename, etc)
-
-        def __getattr__(self, item):
-            # We add these for Python >= 3.4 Loader interface compatibility.
-            if item == 'path':
-                return self.filename
-            elif item == 'name':
-                return self.fullname
-            else:
-                return super(HyLoader, self).__getattr__(item)
-
-        def exec_module(self, module, fullname=None):
-            fullname = self._fix_name(fullname)
-            code = self.get_code(fullname)
-            eval(code, module.__dict__)
-
-        def load_module(self, fullname=None):
-            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
-            source and the option to not run `self.exec_module`."""
-            fullname = self._fix_name(fullname)
-            ext_type = self.etc[0]
-            mod_type = self.etc[2]
-            mod = None
-            pkg_path = os.path.join(self.filename, '__init__.hy')
-            if ext_type == '.hy' or (
-                    mod_type == imp.PKG_DIRECTORY and
-                    os.path.isfile(pkg_path)):
-
-                was_in_sys = fullname in sys.modules
-                if was_in_sys:
-                    mod = sys.modules[fullname]
-                else:
-                    mod = sys.modules.setdefault(
-                        fullname, types.ModuleType(fullname))
-
-                # TODO: Should we set these only when not in `sys.modules`?
-                if mod_type == imp.PKG_DIRECTORY:
-                    mod.__file__ = pkg_path
-                    mod.__path__ = [self.filename]
-                    mod.__package__ = fullname
-                else:
-                    # mod.__path__ = self.filename
-                    mod.__file__ = self.get_filename(fullname)
-                    mod.__package__ = '.'.join(fullname.split('.')[:-1])
-
-                mod.__name__ = fullname
-
-                try:
-                    self.exec_module(mod, fullname=fullname)
-                except Exception:
-                    # Follow Python 2.7 logic and only remove a new, bad
-                    # module; otherwise, leave the old--and presumably
-                    # good--module in there.
-                    if not was_in_sys:
-                        del sys.modules[fullname]
-                    raise
-
-            if mod is None:
-                self._reopen()
-                try:
-                    mod = imp.load_module(fullname, self.file, self.filename,
-                                          self.etc)
-                finally:
-                    if self.file:
-                        self.file.close()
-
-            mod.__loader__ = self
-            return mod
-
-        def _reopen(self):
-            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
-            source"""
-            if self.file and self.file.closed:
-                ext_type = self.etc[0]
-                if ext_type == '.hy':
-                    self.file = io.open(self.filename, 'rU', encoding='utf-8')
-                else:
-                    super(HyLoader, self)._reopen()
-
-        def byte_compile_hy(self, fullname=None):
-            fullname = self._fix_name(fullname)
-            if fullname is None:
-                fullname = self.fullname
-
-            hy_source = self.get_source(fullname)
-            hy_tree = hy_parse(hy_source, filename=self.filename)
-
-            with loader_module_obj(self) as module:
-                hy_ast = hy_compile(hy_tree, module)
-
-            code = compile(hy_ast, self.filename, 'exec',
-                           hy_ast_compile_flags)
-
-            if not sys.dont_write_bytecode:
-                try:
-                    hyc_compile(code, module=fullname)
-                except IOError:
-                    pass
-            return code
-
-        def get_code(self, fullname=None):
-            """Same as `pkgutil.ImpLoader`, with an extra check for Hy
-            source"""
-            fullname = self._fix_name(fullname)
-            ext_type = self.etc[0]
-            if ext_type == '.hy':
-                # Looks like we have to manually check for--and update--
-                # the bytecode.
-                t_py = long(os.stat(self.filename).st_mtime)
-                pyc_file = cache_from_source(self.filename)
-                if os.path.isfile(pyc_file):
-                    t_pyc = long(os.stat(pyc_file).st_mtime)
-
-                    if t_pyc is not None and t_pyc >= t_py:
-                        with open(pyc_file, 'rb') as f:
-                            if f.read(4) == imp.get_magic():
-                                t = struct.unpack('<I', f.read(4))[0]
-                                if t == t_py:
-                                    self.code = marshal.load(f)
-
-                if self.code is None:
-                    # There's no existing bytecode, or bytecode timestamp
-                    # is older than the source file's.
-                    self.code = self.byte_compile_hy(fullname)
-
-            if self.code is None:
-                super(HyLoader, self).get_code(fullname=fullname)
-
-            return self.code
-
-        def _get_delegate(self):
-            return HyImporter(self.filename).find_module('__init__')
-
-    class HyImporter(ImpImporter, object):
-        def __init__(self, path=None):
-            # We need to be strict about the types of files this importer will
-            # handle.  To start, if the path is not the current directory in
-            # (represented by '' in `sys.path`), then it must be a supported
-            # file type or a directory.  If it isn't, this importer is not
-            # suitable: throw an exception.
-
-            if path == '' or os.path.isdir(path) or (
-                    os.path.isfile(path) and path.endswith('.hy')):
-                self.path = path
-            else:
-                raise ImportError('Invalid path: {}'.format(path))
-
-        def find_loader(self, fullname):
-            return self.find_module(fullname, path=None)
-
-        def find_module(self, fullname, path=None):
-
-            subname = fullname.split(".")[-1]
-
-            if subname != fullname and self.path is None:
-                return None
-
-            if self.path is None:
-                path = None
-            else:
-                path = [os.path.realpath(self.path)]
-
-            fileobj, file_path, etc = None, None, None
-
-            # The following are excerpts from the later pure Python
-            # implementations of the `imp` module (e.g. in Python 3.6).
-            if path is None:
-                path = sys.path
-
-            for entry in path:
-                if (os.path.isfile(entry) and subname == '__main__' and
-                    entry.endswith('.hy')):
-                    file_path = entry
-                    fileobj = io.open(file_path, 'rU', encoding='utf-8')
-                    etc = ('.hy', 'U', imp.PY_SOURCE)
-                    break
-                else:
-                    file_path = os.path.join(entry, subname)
-                    path_init = os.path.join(file_path, '__init__.hy')
-                    if os.path.isfile(path_init):
-                        fileobj = None
-                        etc = ('', '', imp.PKG_DIRECTORY)
-                        break
-
-                    file_path = file_path + '.hy'
-                    if os.path.isfile(file_path):
-                        fileobj = io.open(file_path, 'rU', encoding='utf-8')
-                        etc = ('.hy', 'U', imp.PY_SOURCE)
-                        break
-            else:
-                try:
-                    fileobj, file_path, etc = imp.find_module(subname, path)
-                except (ImportError, IOError):
-                    return None
-
-            return HyLoader(fullname, file_path, fileobj, etc)
-
-    sys.path_hooks.append(HyImporter)
-    sys.path_importer_cache.clear()
-
-    _py_compile_compile = py_compile.compile
-
-    def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False,
-                    module=None):
-        """Write a Hy file, or code object, to pyc.
-
-        This is a patched version of Python 2.7's `py_compile.compile`.
-
-        Also, it tries its best to write the bytecode file atomically.
-
-        Parameters
-        ----------
-        file_or_code : str or instance of `types.CodeType`
-            A filename for a Hy or Python source file or its corresponding code
-            object.
-        cfile : str, optional
-            The filename to use for the bytecode file.  If `None`, use the
-            standard bytecode filename determined by `cache_from_source`.
-        dfile : str, optional
-            The filename to use for compile-time errors.
-        doraise : bool, default False
-            If `True` raise compilation exceptions; otherwise, ignore them.
-        module : str or types.ModuleType, optional
-            The module, or module name, in which the Hy tree is expanded.
-            Default is the caller's module.
-
-        Returns
-        -------
-        out : str
-            The resulting bytecode file name.  Python 3.x returns this, but
-            Python 2.7 doesn't; this function does for convenience.
-        """
-
-        if isinstance(file_or_code, types.CodeType):
-            codeobject = file_or_code
-            filename = codeobject.co_filename
-        else:
-            filename = file_or_code
-
-            with open(filename, 'rb') as f:
-                source_str = f.read().decode('utf-8')
-
-            try:
-                flags = None
-                if _could_be_hy_src(filename):
-                    hy_tree = hy_parse(source_str, filename=filename)
-
-                    if module is None:
-                        module = inspect.getmodule(inspect.stack()[1][0])
-                    elif not inspect.ismodule(module):
-                        module = importlib.import_module(module)
-
-                    source = hy_compile(hy_tree, module)
-                    flags = hy_ast_compile_flags
-
-                codeobject = compile(source, dfile or filename, 'exec', flags)
-            except Exception as err:
-
-                py_exc = py_compile.PyCompileError(err.__class__, err,
-                                                   dfile or filename)
-                if doraise:
-                    raise py_exc
-                else:
-                    traceback.print_exc()
-                    return
-
-        timestamp = long(os.stat(filename).st_mtime)
-
-        if cfile is None:
-            cfile = cache_from_source(filename)
-
-        f = None
-        try:
-            f = tempfile.NamedTemporaryFile('wb', dir=os.path.split(cfile)[0],
-                                            delete=False)
-            f.write('\0\0\0\0')
-            f.write(struct.pack('<I', timestamp))
-            f.write(marshal.dumps(codeobject))
-            f.flush()
-            f.seek(0, 0)
-            f.write(imp.get_magic())
-
-            # Make sure it's written to disk.
-            f.flush()
-            os.fsync(f.fileno())
-            f.close()
-
-            # Rename won't replace an existing dest on Windows.
-            if os.name == 'nt' and os.path.isfile(cfile):
-                os.unlink(cfile)
-
-            os.rename(f.name, cfile)
-        except OSError:
-            try:
-                if f is not None:
-                    os.unlink(f.name)
-            except OSError:
-                pass
-
-        return cfile
-
-    py_compile.compile = hyc_compile
+# Do this one just in case?
+importlib.invalidate_caches()
 
+# XXX: These aren't truly cross-compliant.
+# They're useful for testing, though.
+HyImporter = importlib.machinery.FileFinder
+HyLoader = importlib.machinery.SourceFileLoader
 
 # We create a separate version of runpy, "runhy", that prefers Hy source over
 # Python.
diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index dea1baf..a823c41 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -18,7 +18,7 @@ from hy.lex import hy_parse
 from hy.errors import HyLanguageError
 from hy.lex.exceptions import PrematureEndOfInput
 from hy.compiler import hy_eval, hy_compile
-from hy.importer import HyLoader, cache_from_source
+from hy.importer import HyLoader
 
 try:
     from importlib import reload
@@ -101,7 +101,7 @@ def test_import_autocompiles():
         f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
         f.flush()
 
-        pyc_path = cache_from_source(f.name)
+        pyc_path = importlib.util.cache_from_source(f.name)
 
         try:
             os.remove(pyc_path)
@@ -144,7 +144,7 @@ def test_reload():
 
     def unlink(filename):
         os.unlink(source)
-        bytecode = cache_from_source(source)
+        bytecode = importlib.util.cache_from_source(source)
         if os.path.isfile(bytecode):
             os.unlink(bytecode)
 
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 835939e..fec9ce0 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -409,9 +409,9 @@ in expansions."
  Additionally, we confirm that `require` statements are executed via loaded bytecode."
 
   (import os sys marshal types)
-  (import [hy.importer [cache-from-source]])
+  (import importlib)
 
-  (setv pyc-file (cache-from-source
+  (setv pyc-file (importlib.util.cache-from-source
                    (os.path.realpath
                      (os.path.join
                        "tests" "resources" "macro_with_require.hy"))))
diff --git a/tests/test_bin.py b/tests/test_bin.py
index 297eade..1af7ffe 100644
--- a/tests/test_bin.py
+++ b/tests/test_bin.py
@@ -10,7 +10,7 @@ import shlex
 import subprocess
 import builtins
 
-from hy.importer import cache_from_source
+from importlib.util import cache_from_source
 
 import pytest
 

From 762e5fad2d11b167031cf39d69d900558751fb87 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 17:27:17 -0400
Subject: [PATCH 158/223] Remove Python 2 support in hy.compiler

---
 hy/compiler.py | 205 ++++++++++++-------------------------------------
 1 file changed, 51 insertions(+), 154 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 3b0d700..2963496 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -14,7 +14,7 @@ from hy.errors import (HyCompileError, HyTypeError, HyLanguageError,
 
 from hy.lex import mangle, unmangle, hy_parse, parse_one_thing, LexException
 
-from hy._compat import (PY3, PY36, PY38, reraise)
+from hy._compat import (PY36, PY38, reraise)
 from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 
 import hy.core
@@ -95,15 +95,12 @@ def calling_module(n=1):
 def ast_str(x, piecewise=False):
     if piecewise:
         return ".".join(ast_str(s) if s else "" for s in x.split("."))
-    x = mangle(x)
-    return x if PY3 else x.encode('UTF8')
+    return mangle(x)
 
 
 _special_form_compilers = {}
 _model_compilers = {}
-_decoratables = (ast.FunctionDef, ast.ClassDef)
-if PY3:
-    _decoratables += (ast.AsyncFunctionDef,)
+_decoratables = (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)
 # _bad_roots are fake special operators, which are used internally
 # by other special forms (e.g., `except` in `try`) but can't be
 # used to construct special forms themselves.
@@ -175,7 +172,7 @@ class Result(object):
     object gets added to a Result object, it gets converted on-the-fly.
     """
     __slots__ = ("imports", "stmts", "temp_variables",
-                 "_expr", "__used_expr", "contains_yield")
+                 "_expr", "__used_expr")
 
     def __init__(self, *args, **kwargs):
         if args:
@@ -186,14 +183,12 @@ class Result(object):
         self.stmts = []
         self.temp_variables = []
         self._expr = None
-        self.contains_yield = False
 
         self.__used_expr = False
 
         # XXX: Make sure we only have AST where we should.
         for kwarg in kwargs:
-            if kwarg not in ["imports", "contains_yield", "stmts", "expr",
-                             "temp_variables"]:
+            if kwarg not in ["imports", "stmts", "expr", "temp_variables"]:
                 raise TypeError(
                     "%s() got an unexpected keyword argument '%s'" % (
                         self.__class__.__name__, kwarg))
@@ -273,9 +268,7 @@ class Result(object):
             if isinstance(var, ast.Name):
                 var.id = new_name
                 var.arg = new_name
-            elif isinstance(var, ast.FunctionDef):
-                var.name = new_name
-            elif PY3 and isinstance(var, ast.AsyncFunctionDef):
+            elif isinstance(var, (ast.FunctionDef, ast.AsyncFunctionDef)):
                 var.name = new_name
             else:
                 raise TypeError("Don't know how to rename a %s!" % (
@@ -311,22 +304,17 @@ class Result(object):
         result.stmts = self.stmts + other.stmts
         result.expr = other.expr
         result.temp_variables = other.temp_variables
-        result.contains_yield = False
-        if self.contains_yield or other.contains_yield:
-            result.contains_yield = True
 
         return result
 
     def __str__(self):
         return (
-            "Result(imports=[%s], stmts=[%s], "
-            "expr=%s, contains_yield=%s)"
-        ) % (
+            "Result(imports=[%s], stmts=[%s], expr=%s)"
+        % (
             ", ".join(ast.dump(x) for x in self.imports),
             ", ".join(ast.dump(x) for x in self.stmts),
-            ast.dump(self.expr) if self.expr else None,
-            self.contains_yield
-        )
+            ast.dump(self.expr) if self.expr else None
+        ))
 
 
 def is_unpack(kind, x):
@@ -386,11 +374,7 @@ class HyASTCompiler(object):
             for stdlib_module in hy.core.STDLIB:
                 mod = importlib.import_module(stdlib_module)
                 for e in map(ast_str, getattr(mod, 'EXPORTS', [])):
-                    if getattr(mod, e) is not getattr(builtins, e, ''):
-                        # Don't bother putting a name in _stdlib if it
-                        # points to a builtin with the same name. This
-                        # prevents pointless imports.
-                        self._stdlib[e] = stdlib_module
+                    self._stdlib[e] = stdlib_module
 
     def get_anon_var(self):
         self.anon_var_count += 1
@@ -454,8 +438,7 @@ class HyASTCompiler(object):
     def _syntax_error(self, expr, message):
         return HySyntaxError(message, expr, self.filename, self.source)
 
-    def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
-                         oldpy_unpack=False):
+    def _compile_collect(self, exprs, with_kwargs=False, dict_display=False):
         """Collect the expression contexts from a list of compiled expression.
 
         This returns a list of the expression contexts, and the sum of the
@@ -465,34 +448,18 @@ class HyASTCompiler(object):
         compiled_exprs = []
         ret = Result()
         keywords = []
-        oldpy_starargs = None
-        oldpy_kwargs = None
 
         exprs_iter = iter(exprs)
         for expr in exprs_iter:
 
-            if not PY3 and oldpy_unpack and is_unpack("iterable", expr):
-                if oldpy_starargs:
-                    raise self._syntax_error(expr,
-                        "Pythons < 3.5 allow only one `unpack-iterable` per call")
-                oldpy_starargs = self.compile(expr[1])
-                ret += oldpy_starargs
-                oldpy_starargs = oldpy_starargs.force_expr
-
-            elif is_unpack("mapping", expr):
+            if is_unpack("mapping", expr):
                 ret += self.compile(expr[1])
-                if PY3:
-                    if dict_display:
-                        compiled_exprs.append(None)
-                        compiled_exprs.append(ret.force_expr)
-                    elif with_kwargs:
-                        keywords.append(asty.keyword(
-                            expr, arg=None, value=ret.force_expr))
-                elif oldpy_unpack:
-                    if oldpy_kwargs:
-                        raise self._syntax_error(expr,
-                            "Pythons < 3.5 allow only one `unpack-mapping` per call")
-                    oldpy_kwargs = ret.force_expr
+                if dict_display:
+                    compiled_exprs.append(None)
+                    compiled_exprs.append(ret.force_expr)
+                elif with_kwargs:
+                    keywords.append(asty.keyword(
+                        expr, arg=None, value=ret.force_expr))
 
             elif with_kwargs and isinstance(expr, HyKeyword):
                 try:
@@ -516,10 +483,7 @@ class HyASTCompiler(object):
                 ret += self.compile(expr)
                 compiled_exprs.append(ret.force_expr)
 
-        if oldpy_unpack:
-            return compiled_exprs, ret, keywords, oldpy_starargs, oldpy_kwargs
-        else:
-            return compiled_exprs, ret, keywords
+        return compiled_exprs, ret, keywords
 
     def _compile_branch(self, exprs):
         """Make a branch out of an iterable of Result objects
@@ -560,7 +524,7 @@ class HyASTCompiler(object):
             new_name = ast.Subscript(value=name.value, slice=name.slice)
         elif isinstance(name, ast.Attribute):
             new_name = ast.Attribute(value=name.value, attr=name.attr)
-        elif PY3 and isinstance(name, ast.Starred):
+        elif isinstance(name, ast.Starred):
             new_name = ast.Starred(
                 value=self._storeize(expr, name.value, func))
         else:
@@ -646,23 +610,10 @@ class HyASTCompiler(object):
 
     @special("unpack-iterable", [FORM])
     def compile_unpack_iterable(self, expr, root, arg):
-        if not PY3:
-            raise self._syntax_error(expr,
-                "`unpack-iterable` isn't allowed here")
         ret = self.compile(arg)
         ret += asty.Starred(expr, value=ret.force_expr, ctx=ast.Load())
         return ret
 
-    @special([(not PY3, "exec*")], [FORM, maybe(FORM), maybe(FORM)])
-    # Under Python 3, `exec` is a function rather than a statement type, so Hy
-    # doesn't need a special form for it.
-    def compile_exec(self, expr, root, body, globals_, locals_):
-        return asty.Exec(
-            expr,
-            body=self.compile(body).force_expr,
-            globals=self.compile(globals_).force_expr if globals_ is not None else None,
-            locals=self.compile(locals_).force_expr if locals_ is not None else None)
-
     @special("do", [many(FORM)])
     def compile_do(self, expr, root, body):
         return self._compile_branch(body)
@@ -677,9 +628,6 @@ class HyASTCompiler(object):
             exc = exc.force_expr
 
         if cause is not None:
-            if not PY3:
-                raise self._syntax_error(expr,
-                    "raise from only supported in python 3")
             cause = self.compile(cause)
             ret += cause
             cause = cause.force_expr
@@ -735,35 +683,17 @@ class HyASTCompiler(object):
 
         returnable = Result(
             expr=asty.Name(expr, id=return_var.id, ctx=ast.Load()),
-            temp_variables=[return_var],
-            contains_yield=body.contains_yield)
+            temp_variables=[return_var])
         body += body.expr_as_stmt() if orelse else asty.Assign(
             expr, targets=[return_var], value=body.force_expr)
         body = body.stmts or [asty.Pass(expr)]
 
-        if PY3:
-            # Python 3.3 features a merge of TryExcept+TryFinally into Try.
-            x = asty.Try(
-                expr,
-                body=body,
-                handlers=handlers,
-                orelse=orelse,
-                finalbody=finalbody)
-        elif finalbody and handlers:
-            x = asty.TryFinally(
-                expr,
-                body=[asty.TryExcept(
-                    expr,
-                    body=body,
-                    handlers=handlers,
-                    orelse=orelse)],
-                finalbody=finalbody)
-        elif finalbody:
-            x = asty.TryFinally(
-                expr, body=body, finalbody=finalbody)
-        else:
-            x = asty.TryExcept(
-                expr, body=body, handlers=handlers, orelse=orelse)
+        x = asty.Try(
+            expr,
+            body=body,
+            handlers=handlers,
+            orelse=orelse,
+            finalbody=finalbody)
         return handler_results + x + returnable
 
     def _compile_catch_expression(self, expr, var, exceptions, body):
@@ -780,9 +710,7 @@ class HyASTCompiler(object):
 
         name = None
         if len(exceptions) == 2:
-            name = exceptions[0]
-            name = (ast_str(name) if PY3
-                    else self._storeize(name, self.compile(name)))
+            name = ast_str(exceptions[0])
 
         exceptions_list = exceptions[-1] if exceptions else HyList()
         if isinstance(exceptions_list, HyList):
@@ -896,19 +824,19 @@ class HyASTCompiler(object):
             msg = self.compile(msg).force_expr
         return ret + asty.Assert(expr, test=e, msg=msg)
 
-    @special(["global", (PY3, "nonlocal")], [oneplus(SYM)])
+    @special(["global", "nonlocal"], [oneplus(SYM)])
     def compile_global_or_nonlocal(self, expr, root, syms):
         node = asty.Global if root == "global" else asty.Nonlocal
         return node(expr, names=list(map(ast_str, syms)))
 
     @special("yield", [maybe(FORM)])
     def compile_yield_expression(self, expr, root, arg):
-        ret = Result(contains_yield=(not PY3))
+        ret = Result()
         if arg is not None:
             ret += self.compile(arg)
         return ret + asty.Yield(expr, value=ret.force_expr)
 
-    @special([(PY3, "yield-from"), (PY3, "await")], [FORM])
+    @special(["yield-from", "await"], [FORM])
     def compile_yield_from_or_await_expression(self, expr, root, arg):
         ret = Result() + self.compile(arg)
         node = asty.YieldFrom if root == "yield-from" else asty.Await
@@ -987,7 +915,7 @@ class HyASTCompiler(object):
         fn.stmts[-1].decorator_list = decs + fn.stmts[-1].decorator_list
         return ret + fn
 
-    @special(["with*", (PY3, "with/a*")],
+    @special(["with*", "with/a*"],
              [brackets(FORM, maybe(FORM)), many(FORM)])
     def compile_with_expression(self, expr, root, args, body):
         thing, ctx = (None, args[0]) if args[1] is None else args
@@ -1011,14 +939,11 @@ class HyASTCompiler(object):
         the_with = node(expr,
                         context_expr=ctx.force_expr,
                         optional_vars=thing,
-                        body=body.stmts)
-
-        if PY3:
-            the_with.items = [ast.withitem(context_expr=ctx.force_expr,
-                                           optional_vars=thing)]
+                        body=body.stmts,
+                        items=[ast.withitem(context_expr=ctx.force_expr,
+                                            optional_vars=thing)])
 
         ret = Result(stmts=[initial_assign]) + ctx + the_with
-        ret.contains_yield = ret.contains_yield or body.contains_yield
         # And make our expression context our temp variable
         expr_name = asty.Name(expr, id=ast_str(var), ctx=ast.Load())
 
@@ -1092,7 +1017,6 @@ class HyASTCompiler(object):
             # The desired comprehension can't be expressed as a
             # real Python comprehension. We'll write it as a nested
             # loop in a function instead.
-            contains_yield = []
             def f(parts):
                 # This function is called recursively to construct
                 # the nested loop.
@@ -1100,8 +1024,6 @@ class HyASTCompiler(object):
                     if is_for:
                         if body:
                             bd = self._compile_branch(body)
-                            if bd.contains_yield:
-                                contains_yield.append(True)
                             return bd + bd.expr_as_stmt()
                         return Result(stmts=[asty.Pass(expr)])
                     if node_class is asty.DictComp:
@@ -1132,9 +1054,7 @@ class HyASTCompiler(object):
                 else:
                     raise ValueError("can't happen")
             if is_for:
-                ret = f(parts)
-                ret.contains_yield = bool(contains_yield)
-                return ret
+                return f(parts)
             fname = self.get_anon_var()
             # Define the generator function.
             ret = Result() + asty.FunctionDef(
@@ -1343,12 +1263,11 @@ class HyASTCompiler(object):
              ">>": ast.RShift,
              "|": ast.BitOr,
              "^": ast.BitXor,
-             "&": ast.BitAnd}
-    if PY3:
-        m_ops["@"] = ast.MatMult
+             "&": ast.BitAnd,
+             "@": ast.MatMult}
 
     @special(["+", "*", "|"], [many(FORM)])
-    @special(["-", "/", "&", (PY3, "@")], [oneplus(FORM)])
+    @special(["-", "/", "&", "@"], [oneplus(FORM)])
     @special(["**", "//", "<<", ">>"], [times(2, Inf, FORM)])
     @special(["%", "^"], [times(2, 2, FORM)])
     def compile_maths_expression(self, expr, root, args):
@@ -1407,9 +1326,7 @@ class HyASTCompiler(object):
     def _compile_assign(self, root, name, result):
 
         str_name = "%s" % name
-        if str_name in (["None"] + (["True", "False"] if PY3 else [])):
-            # Python 2 allows assigning to True and False, although
-            # this is rarely wise.
+        if str_name in ("None", "True", "False"):
             raise self._syntax_error(name,
                 "Can't assign to `%s'" % str_name)
 
@@ -1473,13 +1390,12 @@ class HyASTCompiler(object):
             expr, test=cond_compiled.force_expr,
             body=body.stmts or [asty.Pass(expr)],
             orelse=orel.stmts)
-        ret.contains_yield = body.contains_yield
 
         return ret
 
     NASYM = some(lambda x: isinstance(x, HySymbol) and x not in (
         "&optional", "&rest", "&kwonly", "&kwargs"))
-    @special(["fn", "fn*", (PY3, "fn/a")], [
+    @special(["fn", "fn*", "fn/a"], [
         # The starred version is for internal use (particularly, in the
         # definition of `defn`). It ensures that a FunctionDef is
         # produced rather than a Lambda.
@@ -1497,25 +1413,14 @@ class HyASTCompiler(object):
 
         mandatory, optional, rest, kwonly, kwargs = params
         optional, defaults, ret = self._parse_optional_args(optional)
-        if kwonly is not None and not PY3:
-            raise self._syntax_error(params,
-                "&kwonly parameters require Python 3")
         kwonly, kw_defaults, ret2 = self._parse_optional_args(kwonly, True)
         ret += ret2
         main_args = mandatory + optional
 
-        if PY3:
-            # Python 3.4+ requires that args are an ast.arg object, rather
-            # than an ast.Name or bare string.
-            main_args, kwonly, [rest], [kwargs] = (
-                [[x and asty.arg(x, arg=ast_str(x), annotation=None)
-                  for x in o]
-                 for o in (main_args or [], kwonly or [], [rest], [kwargs])])
-        else:
-            main_args = [asty.Name(x, id=ast_str(x), ctx=ast.Param())
-                         for x in main_args]
-            rest = rest and ast_str(rest)
-            kwargs = kwargs and ast_str(kwargs)
+        main_args, kwonly, [rest], [kwargs] = (
+            [[x and asty.arg(x, arg=ast_str(x), annotation=None)
+              for x in o]
+             for o in (main_args or [], kwonly or [], [rest], [kwargs])])
 
         args = ast.arguments(
             args=main_args, defaults=defaults,
@@ -1529,13 +1434,7 @@ class HyASTCompiler(object):
             return ret + asty.Lambda(expr, args=args, body=body.force_expr)
 
         if body.expr:
-            if body.contains_yield and not PY3:
-                # Prior to PEP 380 (introduced in Python 3.3)
-                # generators may not have a value in a return
-                # statement.
-                body += body.expr_as_stmt()
-            else:
-                body += asty.Return(body.expr, value=body.expr)
+            body += asty.Return(body.expr, value=body.expr)
 
         name = self.get_anon_var()
 
@@ -1581,7 +1480,7 @@ class HyASTCompiler(object):
         base_list, docstring, attrs, body = rest or ([[]], None, None, [])
 
         bases_expr, bases, keywords = (
-            self._compile_collect(base_list[0], with_kwargs=PY3))
+            self._compile_collect(base_list[0], with_kwargs=True))
 
         bodyr = Result()
 
@@ -1750,12 +1649,10 @@ class HyASTCompiler(object):
         # a typecheck, eg (type :foo)
         with_kwargs = root not in (
             "type", "HyKeyword", "keyword", "name", "keyword?", "identity")
-        args, ret, keywords, oldpy_star, oldpy_kw = self._compile_collect(
-            args, with_kwargs, oldpy_unpack=True)
+        args, ret, keywords = self._compile_collect(args, with_kwargs)
 
         return func + ret + asty.Call(
-            expr, func=func.expr, args=args, keywords=keywords,
-            starargs=oldpy_star, kwargs=oldpy_kw)
+            expr, func=func.expr, args=args, keywords=keywords)
 
     @builds_model(HyInteger, HyFloat, HyComplex)
     def compile_numeric_literal(self, x):
@@ -1807,7 +1704,7 @@ class HyASTCompiler(object):
         if type(string) is HyString and string.is_format:
             # This is a format string (a.k.a. an f-string).
             return self._format_string(string, str(string))
-        node = asty.Bytes if PY3 and type(string) is HyBytes else asty.Str
+        node = asty.Bytes if type(string) is HyBytes else asty.Str
         f = bytes if type(string) is HyBytes else str
         return node(string, s=f(string))
 

From 7ba140725706c07aca0b43bef2c40275f4b99f89 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 16:58:52 -0400
Subject: [PATCH 159/223] Remove hy._compat.PY3

---
 hy/_compat.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hy/_compat.py b/hy/_compat.py
index 4b7d471..70a3187 100644
--- a/hy/_compat.py
+++ b/hy/_compat.py
@@ -4,7 +4,6 @@
 
 import sys
 
-PY3 = sys.version_info[0] >= 3
 PY36 = sys.version_info >= (3, 6)
 PY37 = sys.version_info >= (3, 7)
 PY38 = sys.version_info >= (3, 8)

From 6af6a2945a6740590582e3775f0da6178dbfa701 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 17:09:24 -0400
Subject: [PATCH 160/223] Remove `if-python2` and its uses

---
 hy/contrib/profile.hy               |  4 +-
 hy/core/bootstrap.hy                |  7 ---
 hy/core/language.hy                 | 82 +++++------------------------
 tests/native_tests/language.hy      |  4 +-
 tests/native_tests/native_macros.hy |  5 --
 5 files changed, 16 insertions(+), 86 deletions(-)

diff --git a/hy/contrib/profile.hy b/hy/contrib/profile.hy
index 8ff2ed6..cea3435 100644
--- a/hy/contrib/profile.hy
+++ b/hy/contrib/profile.hy
@@ -19,9 +19,7 @@
   `(do
      (import cProfile pstats)
 
-     (if-python2
-       (import [StringIO [StringIO]])
-       (import [io [StringIO]]))
+     (import [io [StringIO]])
 
      (setv ~g!hy-pr (.Profile cProfile))
      (.enable ~g!hy-pr)
diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy
index 0d3d737..d5b7e19 100644
--- a/hy/core/bootstrap.hy
+++ b/hy/core/bootstrap.hy
@@ -77,10 +77,3 @@
   (if (not (isinstance lambda-list hy.HyList))
     (macro-error name "defn/a takes a parameter list as second argument"))
   `(setv ~name (fn/a ~lambda-list ~@body)))
-
-(defmacro if-python2 [python2-form python3-form]
-  "If running on python2, execute python2-form, else, execute python3-form"
-  (import sys)
-  (if (< (get sys.version_info 0) 3)
-    python2-form
-    python3-form))
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 687bd5c..c08c00c 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -11,6 +11,7 @@
 (import [fractions [Fraction :as fraction]])
 (import operator)  ; shadow not available yet
 (import sys)
+(import [collections.abc :as cabc])
 (import [hy.models [HySymbol HyKeyword]])
 (import [hy.lex [tokenize mangle unmangle read read-str]])
 (import [hy.lex.exceptions [LexException PrematureEndOfInput]])
@@ -20,10 +21,6 @@
 
 (require [hy.core.bootstrap [*]])
 
-(if-python2
-  (import [collections :as cabc])
-  (import [collections.abc :as cabc]))
-
 (defn butlast [coll]
   "Return an iterator of all but the last item in `coll`."
   (drop-last 1 coll))
@@ -85,47 +82,12 @@ If the second argument `codegen` is true, generate python code instead."
        (yield val)
        (.add seen val)))))
 
-(if-python2
- (setv
-   remove itertools.ifilterfalse
-   zip-longest itertools.izip_longest
-   ;; not builtin in Python3
-   reduce reduce
-   ;; hy is more like Python3
-   filter itertools.ifilter
-   input raw_input
-   map itertools.imap
-   range xrange
-   zip itertools.izip)
- (setv
-   remove itertools.filterfalse
-   zip-longest itertools.zip_longest
-   ;; was builtin in Python2
-   reduce functools.reduce
-   ;; Someone can import these directly from `hy.core.language`;
-   ;; we'll make some duplicates.
-   filter filter
-   input input
-   map map
-   range range
-   zip zip))
-
-(if-python2
-  (defn exec [$code &optional $globals $locals]
-    "Execute Python code.
-
-The parameter names contain weird characters to discourage calling this
-function with keyword arguments, which isn't supported by Python 3's `exec`."
-    (if
-      (none? $globals) (do
-        (setv frame (._getframe sys (int 1)))
-        (try
-          (setv $globals frame.f_globals  $locals frame.f_locals)
-          (finally (del frame))))
-      (none? $locals)
-        (setv $locals $globals))
-    (exec* $code $globals $locals))
-  (setv exec exec))
+(setv
+  remove itertools.filterfalse
+  zip-longest itertools.zip_longest
+  ;; was builtin in Python2
+  reduce functools.reduce
+  accumulate itertools.accumulate)
 
 ;; infinite iterators
 (setv
@@ -151,18 +113,6 @@ function with keyword arguments, which isn't supported by Python 3's `exec`."
   permutations itertools.permutations
   product itertools.product)
 
-;; also from itertools, but not in Python2, and without func option until 3.3
-(defn accumulate [iterable &optional [func operator.add]]
-  "Accumulate `func` on `iterable`.
-
-Return series of accumulated sums (or other binary function results)."
-  (setv it (iter iterable)
-        total (next it))
-  (yield total)
-  (for [element it]
-    (setv total (func total element))
-    (yield total)))
-
 (defn drop [count coll]
   "Drop `count` elements from `coll` and yield back the rest."
   (islice coll count None))
@@ -389,15 +339,11 @@ with overlap."
 
 (defn string [x]
   "Cast `x` as the current python version's string implementation."
-  (if-python2
-   (unicode x)
-   (str x)))
+  (str x))
 
 (defn string? [x]
   "Check if `x` is a string."
-  (if-python2
-    (isinstance x (, str unicode))
-    (isinstance x str)))
+  (isinstance x str))
 
 (defn take [count coll]
   "Take `count` elements from `coll`."
@@ -456,11 +402,11 @@ Even objects with the __name__ magic will work."
 (setv EXPORTS
   '[*map accumulate butlast calling-module calling-module-name chain coll?
     combinations comp complement compress constantly count cycle dec distinct
-    disassemble drop drop-last drop-while empty? eval even? every? exec first
-    filter flatten float? fraction gensym group-by identity inc input instance?
+    disassemble drop drop-last drop-while empty? eval even? every? first
+    flatten float? fraction gensym group-by identity inc instance?
     integer integer? integer-char? interleave interpose islice iterable?
     iterate iterator? juxt keyword keyword? last list? macroexpand
-    macroexpand-1 mangle map merge-with multicombinations name neg? none? nth
-    numeric? odd? partition permutations pos? product range read read-str
+    macroexpand-1 mangle merge-with multicombinations name neg? none? nth
+    numeric? odd? partition permutations pos? product read read-str
     remove repeat repeatedly rest reduce second some string string? symbol?
-    take take-nth take-while tuple? unmangle xor tee zero? zip zip-longest])
+    take take-nth take-while tuple? unmangle xor tee zero? zip-longest])
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 63b7599..931e0d0 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1579,9 +1579,7 @@ macros()
 
 (defn test-read []
   "NATIVE: test that read takes something for stdin and reads"
-  (if-python2
-    (import [StringIO [StringIO]])
-    (import [io [StringIO]]))
+  (import [io [StringIO]])
   (import [hy.models [HyExpression]])
 
   (setv stdin-buffer (StringIO "(+ 2 2)\n(- 2 2)"))
diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index fec9ce0..61001db 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -140,11 +140,6 @@
 (assert initialized)
 (assert (test-initialized))
 
-(defn test-if-python2 []
-  (import sys)
-  (assert (= (get sys.version_info 0)
-             (if-python2 2 3))))
-
 (defn test-gensym-in-macros []
   (import ast)
   (import [astor.code-gen [to-source]])

From 9b4178ebd0620f4c61d9e72d81af0c1d5d0488ba Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 17:17:39 -0400
Subject: [PATCH 161/223] Remove undocumented fns `integer` and `string`

---
 hy/core/language.hy            | 16 ++++------------
 tests/native_tests/core.hy     |  8 ++++----
 tests/native_tests/language.hy |  5 -----
 3 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/hy/core/language.hy b/hy/core/language.hy
index c08c00c..80cba83 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -201,10 +201,6 @@ If the second argument `codegen` is true, generate python code instead."
   "Perform `isinstance` with reversed arguments."
   (isinstance x klass))
 
-(defn integer [x]
-  "Return Hy kind of integer for `x`."
-  (int x))
-
 (defn integer? [x]
   "Check if `x` is an integer."
   (isinstance x int))
@@ -337,10 +333,6 @@ with overlap."
   "Return the first logical true value of applying `pred` in `coll`, else None."
   (first (filter None (map pred coll))))
 
-(defn string [x]
-  "Cast `x` as the current python version's string implementation."
-  (str x))
-
 (defn string? [x]
   "Check if `x` is a string."
   (isinstance x str))
@@ -378,7 +370,7 @@ Strings numbers and even objects with the __name__ magic will work."
           (HyKeyword (unmangle value))
           (try
             (unmangle (.__name__ value))
-            (except [] (HyKeyword (string value)))))))
+            (except [] (HyKeyword (str value)))))))
 
 (defn name [value]
   "Convert `value` to a string.
@@ -391,7 +383,7 @@ Even objects with the __name__ magic will work."
           (unmangle value)
           (try
             (unmangle (. value __name__))
-            (except [] (string value))))))
+            (except [] (str value))))))
 
 (defn xor [a b]
   "Perform exclusive or between `a` and `b`."
@@ -404,9 +396,9 @@ Even objects with the __name__ magic will work."
     combinations comp complement compress constantly count cycle dec distinct
     disassemble drop drop-last drop-while empty? eval even? every? first
     flatten float? fraction gensym group-by identity inc instance?
-    integer integer? integer-char? interleave interpose islice iterable?
+    integer? integer-char? interleave interpose islice iterable?
     iterate iterator? juxt keyword keyword? last list? macroexpand
     macroexpand-1 mangle merge-with multicombinations name neg? none? nth
     numeric? odd? partition permutations pos? product read read-str
-    remove repeat repeatedly rest reduce second some string string? symbol?
+    remove repeat repeatedly rest reduce second some string? symbol?
     take take-nth take-while tuple? unmangle xor tee zero? zip-longest])
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index 787a390..0618766 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -285,7 +285,7 @@ result['y in globals'] = 'y' in globals()")
   (setv s3 (gensym "xx"))
   (assert (= 0 (.find s2 "_xx\uffff")))
   (assert (not (= s2 s3)))
-  (assert (not (= (string s2) (string s3)))))
+  (assert (not (= (str s2) (str s3)))))
 
 (defn test-identity []
   "NATIVE: testing the identity function"
@@ -322,8 +322,8 @@ result['y in globals'] = 'y' in globals()")
   (assert-true (integer? 0))
   (assert-true (integer? 3))
   (assert-true (integer? -3))
-  (assert-true (integer? (integer "-3")))
-  (assert-true (integer? (integer 3)))
+  (assert-true (integer? (int "-3")))
+  (assert-true (integer? (int 3)))
   (assert-false (integer? 4.2))
   (assert-false (integer? None))
   (assert-false (integer? "foo")))
@@ -332,7 +332,7 @@ result['y in globals'] = 'y' in globals()")
   "NATIVE: testing the integer-char? function"
   (assert-true (integer-char? "1"))
   (assert-true (integer-char? "-1"))
-  (assert-true (integer-char? (str (integer 300))))
+  (assert-true (integer-char? (str (int 300))))
   (assert-false (integer-char? "foo"))
   (assert-false (integer-char? None)))
 
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 931e0d0..5593d01 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1469,11 +1469,6 @@ cee\"} dee" "ey bee\ncee dee"))
   (assert (= y [5])))
 
 
-(defn test-string []
-  (assert (string? (string "a")))
-  (assert (string? (string 1)))
-  (assert (= u"unicode" (string "unicode"))))
-
 (defn test-del []
   "NATIVE: Test the behavior of del"
   (setv foo 42)

From 081c22d50b993499dbb05bd2ba0f70e1db053389 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 20 May 2019 17:21:34 -0400
Subject: [PATCH 162/223] Update NEWS

---
 NEWS.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 94db065..00103d4 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1,5 +1,12 @@
 .. default-role:: code
 
+Unreleased
+==============================
+
+Removals
+------------------------------
+* Python 2 is no longer supported.
+
 0.17.0
 ==============================
 

From 9914e9010c753c23fc7cf96c0de0dd2988b30b92 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 23 May 2019 13:20:44 -0400
Subject: [PATCH 163/223] Update the docs for removing Python 2 support

Some of the example output may still be from Python 2.
---
 docs/conf.py                |  3 +--
 docs/language/api.rst       | 15 ++-------------
 docs/language/core.rst      | 18 ++----------------
 docs/language/internals.rst | 10 ++++------
 docs/language/syntax.rst    | 12 ++++--------
 docs/style-guide.rst        |  1 -
 docs/tutorial.rst           |  3 ---
 7 files changed, 13 insertions(+), 49 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 682dbcf..f4ae994 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -52,5 +52,4 @@ html_context = dict(
     hy_descriptive_version = hy_descriptive_version)
 
 intersphinx_mapping = dict(
-    py2 = ('https://docs.python.org/2/', None),
-    py  = ('https://docs.python.org/3/', None))
+    py = ('https://docs.python.org/3/', None))
diff --git a/docs/language/api.rst b/docs/language/api.rst
index 2a690f6..6b243ba 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -564,8 +564,6 @@ requires.
           File "<input>", line 1, in <module>
         TypeError: compare() missing 1 required keyword-only argument: 'keyfn'
 
-    Availability: Python 3.
-
 &kwargs
     Like ``&rest``, but for keyword arugments.
     The following parameter will contain 0 or more keyword arguments.
@@ -1057,7 +1055,7 @@ if / if* / if-not
 ``if / if* / if-not`` respect Python *truthiness*, that is, a *test* fails if it
 evaluates to a "zero" (including values of ``len`` zero, ``None``, and
 ``False``), and passes otherwise, but values with a ``__bool__`` method
-(``__nonzero__`` in Python 2) can overrides this.
+can override this.
 
 The ``if`` macro is for conditionally selecting an expression for evaluation.
 The result of the selected expression becomes the result of the entire ``if``
@@ -1296,19 +1294,12 @@ fact, these forms are implemented as generator functions whenever they
 contain Python statements, with the attendant consequences for calling
 ``return``. By contrast, ``for`` shares the caller's scope.
 
-.. note:: An exception to the above scoping rules occurs on Python 2 for
-   ``lfor`` specifically (and not ``sfor``, ``gfor``, or ``dfor``) when
-   Hy can implement the ``lfor`` as a Python list comprehension. Then,
-   variables will leak to the surrounding scope.
-
 
 nonlocal
 --------
 
 .. versionadded:: 0.11.1
 
-**PYTHON 3.0 AND UP ONLY!**
-
 ``nonlocal`` can be used to mark a symbol as not local to the current scope.
 The parameters are the names of symbols to mark as nonlocal.  This is necessary
 to modify variables through nested ``fn`` scopes:
@@ -1693,7 +1684,7 @@ object (respectively) to provide positional or keywords arguments
     => (f #* [1 2] #** {"c" 3 "d" 4})
     [1, 2, 3, 4]
 
-With Python 3, unpacking is allowed in more contexts, and you can unpack
+Unpacking is allowed in a variety of contexts, and you can unpack
 more than once in one expression (:pep:`3132`, :pep:`448`).
 
 .. code-block:: clj
@@ -2038,8 +2029,6 @@ yield-from
 
 .. versionadded:: 0.9.13
 
-**PYTHON 3.3 AND UP ONLY!**
-
 ``yield-from`` is used to call a subgenerator.  This is useful if you
 want your coroutine to be able to delegate its processes to another
 coroutine, say, if using something fancy like
diff --git a/docs/language/core.rst b/docs/language/core.rst
index 7e42691..cabb420 100644
--- a/docs/language/core.rst
+++ b/docs/language/core.rst
@@ -240,19 +240,6 @@ otherwise ``False``. Return ``True`` if *coll* is empty.
    True
 
 
-.. _exec-fn:
-
-exec
-----
-
-Equivalent to Python 3's built-in function :py:func:`exec`.
-
-.. code-block:: clj
-
-    => (exec "print(a + b)" {"a" 1} {"b" 2})
-    3
-
-
 .. _float?-fn:
 
 float?
@@ -385,8 +372,7 @@ integer?
 
 Usage: ``(integer? x)``
 
-Returns `True` if *x* is an integer. For Python 2, this is
-either ``int`` or ``long``. For Python 3, this is ``int``.
+Returns `True` if *x* is an integer (``int``).
 
 .. code-block:: hy
 
@@ -924,7 +910,7 @@ string?
 
 Usage: ``(string? x)``
 
-Returns ``True`` if *x* is a string.
+Returns ``True`` if *x* is a string (``str``).
 
 .. code-block:: hy
 
diff --git a/docs/language/internals.rst b/docs/language/internals.rst
index 1cd32ad..a89b356 100644
--- a/docs/language/internals.rst
+++ b/docs/language/internals.rst
@@ -120,9 +120,7 @@ HyString
 ~~~~~~~~
 
 ``hy.models.HyString`` represents string literals (including bracket strings),
-which compile down to unicode string literals in Python. ``HyStrings`` inherit
-unicode objects in Python 2, and string objects in Python 3 (and are therefore
-not encoding-dependent).
+which compile down to unicode string literals (``str``) in Python.
 
 ``HyString``\s are immutable.
 
@@ -140,15 +138,15 @@ HyBytes
 ~~~~~~~
 
 ``hy.models.HyBytes`` is like ``HyString``, but for sequences of bytes.
-It inherits from ``bytes`` on Python 3 and ``str`` on Python 2.
+It inherits from ``bytes``.
 
 .. _hy_numeric_models:
 
 Numeric Models
 ~~~~~~~~~~~~~~
 
-``hy.models.HyInteger`` represents integer literals (using the
-``long`` type on Python 2, and ``int`` on Python 3).
+``hy.models.HyInteger`` represents integer literals, using the ``int``
+type.
 
 ``hy.models.HyFloat`` represents floating-point literals.
 
diff --git a/docs/language/syntax.rst b/docs/language/syntax.rst
index 2414499..16ed838 100644
--- a/docs/language/syntax.rst
+++ b/docs/language/syntax.rst
@@ -10,7 +10,7 @@ An identifier consists of a nonempty sequence of Unicode characters that are not
 numeric literals
 ----------------
 
-In addition to regular numbers, standard notation from Python 3 for non-base 10
+In addition to regular numbers, standard notation from Python for non-base 10
 integers is used. ``0x`` for Hex, ``0o`` for Octal, ``0b`` for Binary.
 
 .. code-block:: clj
@@ -60,13 +60,9 @@ Plain string literals support :ref:`a variety of backslash escapes
 literally, prefix the string with ``r``, as in ``r"slash\not"``. Bracket
 strings are always raw strings and don't allow the ``r`` prefix.
 
-Whether running under Python 2 or Python 3, Hy treats all string literals as
-sequences of Unicode characters by default, and allows you to prefix a plain
-string literal (but not a bracket string) with ``b`` to treat it as a sequence
-of bytes. So when running under Python 3, Hy translates ``"foo"`` and
-``b"foo"`` to the identical Python code, but when running under Python 2,
-``"foo"`` is translated to ``u"foo"`` and ``b"foo"`` is translated to
-``"foo"``.
+Like Python, Hy treats all string literals as sequences of Unicode characters
+by default. You may prefix a plain string literal (but not a bracket string)
+with ``b`` to treat it as a sequence of bytes.
 
 Unlike Python, Hy only recognizes string prefixes (``r``, etc.) in lowercase.
 
diff --git a/docs/style-guide.rst b/docs/style-guide.rst
index 68f2eae..167cad4 100644
--- a/docs/style-guide.rst
+++ b/docs/style-guide.rst
@@ -39,7 +39,6 @@ into the making of Hy.
 + Look like a Lisp; DTRT with it (e.g. dashes turn to underscores).
 + We're still Python. Most of the internals translate 1:1 to Python internals.
 + Use Unicode everywhere.
-+ Fix the bad decisions in Python 2 when we can (see ``true_division``).
 + When in doubt, defer to Python.
 + If you're still unsure, defer to Clojure.
 + If you're even more unsure, defer to Common Lisp.
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 96f8f7f..09cbd52 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -25,9 +25,6 @@ This is pretty cool because it means Hy is several things:
    comfort of Python!
  - For everyone: a pleasant language that has a lot of neat ideas!
 
-Now this tutorial assumes you're running Hy on Python 3. So know things
-are a bit different if you're still using Python 2.
-
 
 Basic intro to Lisp for Pythonistas
 ===================================

From ba9b0239c7fa2a02478e8456ed3867c6f2fec0c5 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 4 Jun 2019 16:03:52 -0400
Subject: [PATCH 164/223] Fix crashes on the new Python 3.8 alpha

---
 hy/compiler.py |  3 ++-
 hy/macros.py   | 11 ++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 2963496..7638893 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1061,7 +1061,7 @@ class HyASTCompiler(object):
                 expr,
                 name=fname,
                 args=ast.arguments(
-                    args=[], vararg=None, kwarg=None,
+                    args=[], vararg=None, kwarg=None, posonlyargs=[],
                     kwonlyargs=[], kw_defaults=[], defaults=[]),
                 body=f(parts).stmts,
                 decorator_list=[])
@@ -1425,6 +1425,7 @@ class HyASTCompiler(object):
         args = ast.arguments(
             args=main_args, defaults=defaults,
             vararg=rest,
+            posonlyargs=[],
             kwonlyargs=kwonly, kw_defaults=kw_defaults,
             kwarg=kwargs)
 
diff --git a/hy/macros.py b/hy/macros.py
index 3c91f11..66fc3ff 100644
--- a/hy/macros.py
+++ b/hy/macros.py
@@ -9,7 +9,7 @@ import traceback
 
 from contextlib import contextmanager
 
-from hy._compat import reraise
+from hy._compat import reraise, PY38
 from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value
 from hy.lex import mangle
 from hy.errors import (HyLanguageError, HyMacroExpansionError, HyTypeError,
@@ -398,7 +398,8 @@ def rename_function(func, new_name):
 
     return _fn
 
-code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize',
-                 'flags', 'code', 'consts', 'names', 'varnames',
-                 'filename', 'name', 'firstlineno', 'lnotab', 'freevars',
-                 'cellvars']
+code_obj_args = ['argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize',
+                 'flags', 'code', 'consts', 'names', 'varnames', 'filename', 'name',
+                 'firstlineno', 'lnotab', 'freevars', 'cellvars']
+if not PY38:
+    code_obj_args.remove("posonlyargcount")

From 413648f6bad0e06c7689d8f09c0b8e19431a99df Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 23 May 2019 13:40:26 -0400
Subject: [PATCH 165/223] Remove update-coreteam.hy

It wasn't very useful, especially because it needed manual updates anyway.
---
 scripts/update-coreteam.hy | 41 --------------------------------------
 1 file changed, 41 deletions(-)
 delete mode 100644 scripts/update-coreteam.hy

diff --git a/scripts/update-coreteam.hy b/scripts/update-coreteam.hy
deleted file mode 100644
index 6e1036f..0000000
--- a/scripts/update-coreteam.hy
+++ /dev/null
@@ -1,41 +0,0 @@
-;; Copyright 2019 the authors.
-;; This file is part of Hy, which is free software licensed under the Expat
-;; license. See the LICENSE.
-
-;; You need to install the requests package first
-
-(import os.path)
-(import requests)
-
-
-(setv *api-url* "https://api.github.com/{}")
-(setv *rst-format* "* `{} <{}>`_")
-(setv *missing-names* {"khinsen" "Konrad Hinsen"})
-;; We have three concealed members on the hylang organization
-;; and GitHub only shows public members if the requester is not
-;; an owner of the organization.
-(setv *concealed-members* [(, "aldeka" "Karen Rustad")
-                           (, "tuturto" "Tuukka Turto")
-                           (, "cndreisbach" "Clinton N. Dreisbach")])
-
-(defn get-dev-name [login]
-  (setv name (get (.json (requests.get (.format *api-url* (+ "users/" login)))) "name"))
-  (if-not name
-    (.get *missing-names* login)
-    name))
-
-(setv coredevs (requests.get (.format *api-url* "orgs/hylang/members")))
-
-(setv result (set))
-(for [dev (.json coredevs)]
-  (result.add (.format *rst-format* (get-dev-name (get dev "login"))
-                       (get dev "html_url"))))
-
-(for [(, login name) *concealed-members*]
-  (result.add (.format *rst-format* name (+ "https://github.com/" login))))
-
-(setv filename (os.path.abspath (os.path.join os.path.pardir
-                                              "docs" "coreteam.rst")))
-
-(with [fobj (open filename "w+")]
-  (fobj.write (+ (.join "\n" result) "\n")))

From 704983ed4427189e13af487ebf65339b852dfe18 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 27 May 2019 13:23:35 -0400
Subject: [PATCH 166/223] Clean up coreteam.rst

---
 docs/coreteam.rst | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/docs/coreteam.rst b/docs/coreteam.rst
index b6ff007..cd9a48e 100644
--- a/docs/coreteam.rst
+++ b/docs/coreteam.rst
@@ -1,16 +1,11 @@
-* `Julien Danjou <https://github.com/jd>`_
-* `Morten Linderud <https://github.com/Foxboron>`_
-* `J Kenneth King <https://github.com/agentultra>`_
-* `Gergely Nagy <https://github.com/algernon>`_
-* `Tuukka Turto <https://github.com/tuturto>`_
-* `Karen Rustad <https://github.com/aldeka>`_
-* `Abhishek L <https://github.com/theanalyst>`_
-* `Christopher Allan Webber <https://github.com/cwebber>`_
-* `Konrad Hinsen <https://github.com/khinsen>`_
-* `Will Kahn-Greene <https://github.com/willkg>`_
-* `Paul Tagliamonte <https://github.com/paultag>`_
+* `Kodi B. Arfer <https://github.com/Kodiologist>`_
 * `Nicolas Dandrimont <https://github.com/olasd>`_
-* `Berker Peksag <https://github.com/berkerpeksag>`_
-* `Clinton N. Dreisbach <https://github.com/cndreisbach>`_
-* `han semaj <https://github.com/microamp>`_
-* `Kodi Arfer <https://github.com/Kodiologist>`_
+* `Julien Danjou <https://github.com/jd>`_
+* `Rob Day <https://github.com/rkday>`_
+* `Simon Gomizelj <https://github.com/vodik>`_
+* `Ryan Gonzalez <https://github.com/refi64>`_
+* `Abhishek Lekshmanan <https://github.com/theanalyst>`_
+* `Morten Linderud <https://github.com/Foxboron>`_
+* `Matthew Odendahl <https://github.com/gilch>`_
+* `Paul Tagliamonte <https://github.com/paultag>`_
+* `Brandon T. Willard <https://github.com/brandonwillard>`_

From d547610adbc35fef28400ef6874cf563e3ca7720 Mon Sep 17 00:00:00 2001
From: Aaron Schumacher <ajschumacher@gmail.com>
Date: Tue, 25 Jun 2019 11:52:33 -0400
Subject: [PATCH 167/223] typo: missing "a"

---
 docs/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 09cbd52..cb2b272 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -535,7 +535,7 @@ Macros
 ======
 
 One really powerful feature of Hy are macros. They are small functions that are
-used to generate code (or data). When program written in Hy is started, the
+used to generate code (or data). When a program written in Hy is started, the
 macros are executed and their output is placed in the program source. After this,
 the program starts executing normally. Very simple example:
 

From 36708e8e996700da256943b3e8162a29fa381473 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 10 Jun 2019 16:12:46 -0400
Subject: [PATCH 168/223] Fix a test for Python 3.8.0b1

`int`, among other types, no longer has a `__str__` method, so `(str '3)` now returns "(HyInteger 3)" instead of "3".
---
 tests/native_tests/native_macros.hy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/native_tests/native_macros.hy b/tests/native_tests/native_macros.hy
index 61001db..d15a686 100644
--- a/tests/native_tests/native_macros.hy
+++ b/tests/native_tests/native_macros.hy
@@ -386,7 +386,7 @@ in expansions."
   ;; Now, let's use a `require`d macro that depends on another macro defined only
   ;; in this scope.
   (defmacro local-test-macro [x]
-    (.format "This is the local version of `nonlocal-test-macro` returning {}!" x))
+    (.format "This is the local version of `nonlocal-test-macro` returning {}!" (int x)))
 
   (assert (= "This is the local version of `nonlocal-test-macro` returning 3!"
              (test-module-macro-2 3)))

From e436d9dd4df4df65a422868f99374fdef40753bc Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 21 Jun 2019 15:45:04 -0400
Subject: [PATCH 169/223] Remove an obsolete check for `importlib.reload`

---
 tests/importer/test_importer.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index a823c41..a604a73 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -10,6 +10,7 @@ import runpy
 import importlib
 
 from fractions import Fraction
+from importlib import reload
 
 import pytest
 
@@ -20,11 +21,6 @@ from hy.lex.exceptions import PrematureEndOfInput
 from hy.compiler import hy_eval, hy_compile
 from hy.importer import HyLoader
 
-try:
-    from importlib import reload
-except ImportError:
-    from imp import reload
-
 
 def test_basics():
     "Make sure the basics of the importer work"

From eb265181bf36cfd1d64bc180758b5bb92a4204f8 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 28 Jun 2019 13:41:14 -0400
Subject: [PATCH 170/223] Add a test for a previously fixed reloading bug

---
 tests/importer/test_importer.py | 12 ++++++++++++
 tests/resources/hello_world.hy  |  1 +
 2 files changed, 13 insertions(+)
 create mode 100644 tests/resources/hello_world.hy

diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py
index a604a73..aba89e4 100644
--- a/tests/importer/test_importer.py
+++ b/tests/importer/test_importer.py
@@ -234,6 +234,18 @@ def test_reload():
         unlink(source)
 
 
+def test_reload_reexecute(capsys):
+    """A module is re-executed when it's reloaded, even if it's
+    unchanged.
+
+    https://github.com/hylang/hy/issues/712"""
+    import tests.resources.hello_world
+    assert capsys.readouterr().out == 'hello world\n'
+    assert capsys.readouterr().out == ''
+    reload(tests.resources.hello_world)
+    assert capsys.readouterr().out == 'hello world\n'
+
+
 def test_circular():
     """Test circular imports by creating a temporary file/module that calls a
     function that imports itself."""
diff --git a/tests/resources/hello_world.hy b/tests/resources/hello_world.hy
new file mode 100644
index 0000000..a566b8a
--- /dev/null
+++ b/tests/resources/hello_world.hy
@@ -0,0 +1 @@
+(print "hello world")

From 308bedbebe3e11714b444a7f5be406dd08240608 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 2 Jul 2019 12:02:03 -0400
Subject: [PATCH 171/223] Remove uses of `defclass` attribute lists

---
 hy/contrib/multi.hy                   | 12 ++++++------
 hy/contrib/sequences.hy               | 16 +++++++--------
 tests/native_tests/contrib/hy_repr.hy |  6 +++---
 tests/native_tests/core.hy            |  2 +-
 tests/native_tests/defclass.hy        | 28 +++++++++++++--------------
 tests/native_tests/language.hy        | 26 ++++++++++++-------------
 tests/native_tests/mathematics.hy     |  7 +++----
 tests/native_tests/operators.hy       | 16 +++++++--------
 tests/native_tests/py36_only_tests.hy |  4 ++--
 tests/native_tests/with_decorator.hy  |  2 +-
 tests/resources/pydemo.hy             |  6 +++---
 tests/test_hy2py.py                   |  2 +-
 12 files changed, 63 insertions(+), 64 deletions(-)

diff --git a/hy/contrib/multi.hy b/hy/contrib/multi.hy
index 7dba654..1a19fc9 100644
--- a/hy/contrib/multi.hy
+++ b/hy/contrib/multi.hy
@@ -6,11 +6,11 @@
 (import [collections [defaultdict]]
         [hy [HyExpression HyList HyString]])
 
-(defclass MultiDispatch [object] [
+(defclass MultiDispatch [object]
 
-  _fns (defaultdict dict)
+  (setv _fns (defaultdict dict))
 
-  __init__ (fn [self f]
+  (defn __init__ [self f]
     (setv self.f f)
     (setv self.__doc__ f.__doc__)
     (unless (in f.__name__ (.keys (get self._fns f.__module__)))
@@ -18,14 +18,14 @@
     (setv values f.__code__.co_varnames)
     (setv (get self._fns f.__module__ f.__name__ values) f))
 
-  fn? (fn [self v args kwargs]
+  (defn fn? [self v args kwargs]
     "Compare the given (checked fn) to the called fn"
     (setv com (+ (list args) (list (.keys kwargs))))
     (and
       (= (len com) (len v))
       (.issubset (frozenset (.keys kwargs)) com)))
 
-  __call__ (fn [self &rest args &kwargs kwargs]
+  (defn __call__ [self &rest args &kwargs kwargs]
     (setv func None)
     (for [[i f] (.items (get self._fns self.f.__module__ self.f.__name__))]
       (when (.fn? self i args kwargs)
@@ -33,7 +33,7 @@
         (break)))
     (if func
       (func #* args #** kwargs)
-      (raise (TypeError "No matching functions with this signature"))))])
+      (raise (TypeError "No matching functions with this signature")))))
 
 (defn multi-decorator [dispatch-fn]
   (setv inner (fn [&rest args &kwargs kwargs]
diff --git a/hy/contrib/sequences.hy b/hy/contrib/sequences.hy
index 94b3236..b794b7a 100644
--- a/hy/contrib/sequences.hy
+++ b/hy/contrib/sequences.hy
@@ -3,12 +3,12 @@
 ;; license. See the LICENSE.
 
 (defclass Sequence []
-  [--init-- (fn [self func]
+  (defn --init-- [self func]
               "initialize a new sequence with a function to compute values"
               (setv (. self func) func)
               (setv (. self cache) [])
               (setv (. self high-water) -1))
-   --getitem-- (fn [self n]
+  (defn --getitem-- [self n]
                  "get nth item of sequence"
                  (if (hasattr n "start")
                    (gfor x (range n.start n.stop (or n.step 1))
@@ -23,7 +23,7 @@
                                (setv (. self high-water) (inc (. self high-water)))
                                (.append (. self cache) (.func self (. self high-water))))
                              (get self n))))))
-   --iter-- (fn [self]
+   (defn --iter-- [self]
               "create iterator for this sequence"
               (setv index 0)
               (try (while True
@@ -31,7 +31,7 @@
                      (setv index (inc index)))
                    (except [IndexError]
                      (return))))
-   --len-- (fn [self]
+   (defn --len-- [self]
              "length of the sequence, dangerous for infinite sequences"
              (setv index (. self high-water))
              (try (while True
@@ -39,17 +39,17 @@
                     (setv index (inc index)))
                   (except [IndexError]
                     (len (. self cache)))))
-   max-items-in-repr 10
-   --str-- (fn [self]
+   (setv max-items-in-repr 10)
+   (defn --str-- [self]
              "string representation of this sequence"
              (setv items (list (take (inc self.max-items-in-repr) self)))
              (.format (if (> (len items) self.max-items-in-repr)
                         "[{0}, ...]"
                         "[{0}]")
                       (.join ", " (map str items))))
-   --repr-- (fn [self]
+   (defn --repr-- [self]
               "string representation of this sequence"
-              (.--str-- self))])
+              (.--str-- self)))
 
 (defmacro seq [param &rest seq-code]
   `(Sequence (fn ~param (do ~@seq-code))))
diff --git a/tests/native_tests/contrib/hy_repr.hy b/tests/native_tests/contrib/hy_repr.hy
index 4a0a1ec..3fa807a 100644
--- a/tests/native_tests/contrib/hy_repr.hy
+++ b/tests/native_tests/contrib/hy_repr.hy
@@ -158,8 +158,8 @@
   (assert (= (hy-repr (C)) "cuddles"))
 
   (defclass Container [object]
-    [__init__ (fn [self value]
-      (setv self.value value))])
+    (defn __init__ [self value]
+      (setv self.value value)))
   (hy-repr-register Container :placeholder "(Container ...)" (fn [x]
     (+ "(Container " (hy-repr x.value) ")")))
   (setv container (Container 5))
@@ -170,5 +170,5 @@
 
 (defn test-hy-repr-fallback []
   (defclass D [object]
-    [__repr__ (fn [self] "cuddles")])
+    (defn __repr__ [self] "cuddles"))
   (assert (= (hy-repr (D)) "cuddles")))
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index 0618766..eb1b1c3 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -300,7 +300,7 @@ result['y in globals'] = 'y' in globals()")
   (assert-requires-num inc)
 
   (defclass X [object]
-    [__add__ (fn [self other] (.format "__add__ got {}" other))])
+    (defn __add__ [self other] (.format "__add__ got {}" other)))
   (assert-equal (inc (X)) "__add__ got 1"))
 
 (defn test-instance []
diff --git a/tests/native_tests/defclass.hy b/tests/native_tests/defclass.hy
index 2563072..8fc76bc 100644
--- a/tests/native_tests/defclass.hy
+++ b/tests/native_tests/defclass.hy
@@ -27,7 +27,7 @@
 (defn test-defclass-attrs []
   "NATIVE: test defclass attributes"
   (defclass A []
-    [x 42])
+    (setv x 42))
   (assert (= A.x 42))
   (assert (= (getattr (A) "x")  42)))
 
@@ -35,9 +35,9 @@
 (defn test-defclass-attrs-fn []
   "NATIVE: test defclass attributes with fn"
   (defclass B []
-    [x 42
-     y (fn [self value]
-         (+ self.x value))])
+    (setv x 42)
+    (setv y (fn [self value]
+         (+ self.x value))))
   (assert (= B.x 42))
   (assert (= (.y (B) 5) 47))
   (setv b (B))
@@ -48,17 +48,17 @@
 (defn test-defclass-dynamic-inheritance []
   "NATIVE: test defclass with dynamic inheritance"
   (defclass A [((fn [] (if True list dict)))]
-    [x 42])
+    (setv x 42))
   (assert (isinstance (A) list))
   (defclass A [((fn [] (if False list dict)))]
-    [x 42])
+    (setv x 42))
   (assert (isinstance (A) dict)))
 
 
 (defn test-defclass-no-fn-leak []
   "NATIVE: test defclass attributes with fn"
   (defclass A []
-    [x (fn [] 1)])
+    (setv x (fn [] 1)))
   (try
    (do
     (x)
@@ -68,13 +68,13 @@
 (defn test-defclass-docstring []
   "NATIVE: test defclass docstring"
   (defclass A []
-    [--doc-- "doc string"
-     x 1])
+    (setv --doc-- "doc string")
+    (setv x 1))
   (setv a (A))
   (assert (= a.__doc__ "doc string"))
   (defclass B []
     "doc string"
-    [x 1])
+    (setv x 1))
   (setv b (B))
   (assert (= b.x 1))
   (assert (= b.__doc__ "doc string"))
@@ -82,7 +82,7 @@
     "begin a very long multi-line string to make
      sure that it comes out the way we hope
      and can span 3 lines end."
-    [x 1])
+    (setv x 1))
   (setv mL (MultiLine))
   (assert (= mL.x 1))
   (assert (in "begin" mL.__doc__))
@@ -100,8 +100,8 @@
   "NATIVE: test defclass syntax with properties and methods and side-effects"
   (setv foo 1)
   (defclass A []
-    [x 1
-     y 2]
+    (setv x 1)
+    (setv y 2)
     (global foo)
     (setv foo 2)
     (defn greet [self]
@@ -117,7 +117,7 @@
 (defn test-defclass-implicit-none-for-init []
   "NATIVE: test that defclass adds an implicit None to --init--"
   (defclass A []
-    [--init-- (fn [self] (setv self.x 1) 42)])
+    (setv --init-- (fn [self] (setv self.x 1) 42)))
   (defclass B []
     (defn --init-- [self]
       (setv self.x 2)
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 5593d01..d7bf1da 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -129,8 +129,8 @@
   (assert (none? (setv (get {} "x") 42)))
   (setv l [])
   (defclass Foo [object]
-    [__setattr__ (fn [self attr val]
-      (.append l [attr val]))])
+    (defn __setattr__ [self attr val]
+      (.append l [attr val])))
   (setv x (Foo))
   (assert (none? (setv x.eggs "ham")))
   (assert (not (hasattr x "eggs")))
@@ -443,9 +443,9 @@
 
   (defclass X [object] [])
   (defclass M [object]
-    [meth (fn [self &rest args &kwargs kwargs]
+    (defn meth [self &rest args &kwargs kwargs]
       (.join " " (+ (, "meth") args
-        (tuple (map (fn [k] (get kwargs k)) (sorted (.keys kwargs)))))))])
+        (tuple (map (fn [k] (get kwargs k)) (sorted (.keys kwargs))))))))
 
   (setv x (X))
   (setv m (M))
@@ -1667,7 +1667,7 @@ macros()
 (defn test-underscore_variables []
   ; https://github.com/hylang/hy/issues/1340
   (defclass XYZ []
-    [_42 6])
+    (setv _42 6))
   (setv x (XYZ))
   (assert (= (. x _42) 6)))
 
@@ -1773,24 +1773,24 @@ macros()
 
 (defn test-pep-3115 []
   (defclass member-table [dict]
-    [--init-- (fn [self] (setv self.member-names []))
+    (defn --init-- [self] (setv self.member-names []))
 
-     --setitem-- (fn [self key value]
+    (defn --setitem-- [self key value]
                    (if (not-in key self)
                        (.append self.member-names key))
-                   (dict.--setitem-- self key value))])
+                   (dict.--setitem-- self key value)))
 
   (defclass OrderedClass [type]
-    [--prepare-- (classmethod (fn [metacls name bases] (member-table)))
+    (setv --prepare-- (classmethod (fn [metacls name bases] (member-table))))
 
-     --new-- (fn [cls name bases classdict]
+    (defn --new-- [cls name bases classdict]
                (setv result (type.--new-- cls name bases (dict classdict)))
                (setv result.member-names classdict.member-names)
-               result)])
+               result))
 
   (defclass MyClass [:metaclass OrderedClass]
-    [method1 (fn [self] (pass))
-     method2 (fn [self] (pass))])
+    (defn method1 [self] (pass))
+    (defn method2 [self] (pass)))
 
   (assert (= (. (MyClass) member-names)
              ["__module__" "__qualname__" "method1" "method2"])))
diff --git a/tests/native_tests/mathematics.hy b/tests/native_tests/mathematics.hy
index 8ad8797..23c9045 100644
--- a/tests/native_tests/mathematics.hy
+++ b/tests/native_tests/mathematics.hy
@@ -34,7 +34,7 @@
   "NATIVE: test that unary + calls __pos__"
 
   (defclass X [object]
-    [__pos__ (fn [self] "called __pos__")])
+    (defn __pos__ [self] "called __pos__"))
   (assert (= (+ (X)) "called __pos__"))
 
   ; Make sure the shadowed version works, too.
@@ -159,8 +159,7 @@
 
 
 (defclass HyTestMatrix [list]
-  [--matmul--
-   (fn [self other]
+  (defn --matmul-- [self other]
      (setv n (len self)
            m (len (. other [0]))
            result [])
@@ -173,7 +172,7 @@
                               (. other [k] [j]))))
          (.append result-row dot-product))
        (.append result result-row))
-     result)])
+     result))
 
 (setv first-test-matrix (HyTestMatrix [[1 2 3]
                                        [4 5 6]
diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy
index 6f2ea4a..7c9b18b 100644
--- a/tests/native_tests/operators.hy
+++ b/tests/native_tests/operators.hy
@@ -34,7 +34,7 @@
 
   (assert (= (f) 0))
 
-  (defclass C [object] [__pos__ (fn [self] "called __pos__")])
+  (defclass C [object] (defn __pos__ [self] "called __pos__"))
   (assert (= (f (C)) "called __pos__"))
 
   (assert (= (f 1 2) 3))
@@ -101,9 +101,9 @@
 
 
 (op-and-shadow-test @
-  (defclass C [object] [
-    __init__ (fn [self content] (setv self.content content))
-    __matmul__ (fn [self other] (C (+ self.content other.content)))])
+  (defclass C [object]
+    (defn __init__ [self content] (setv self.content content))
+    (defn __matmul__ [self other] (C (+ self.content other.content))))
   (forbid (f))
   (assert (do (setv c (C "a")) (is (f c) c)))
   (assert (= (. (f (C "b") (C "c")) content) "bc"))
@@ -171,11 +171,11 @@
 
   ; Make sure chained comparisons use `and`, not `&`.
   ; https://github.com/hylang/hy/issues/1191
-  (defclass C [object] [
-    __init__ (fn [self x]
+  (defclass C [object]
+    (defn __init__ [self x]
       (setv self.x x))
-    __lt__ (fn [self other]
-      self.x)])
+    (defn __lt__ [self other]
+      self.x))
   (assert (= (f (C "a") (C "b") (C "c")) "b")))
 
 
diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy
index 40de6d9..7f5908a 100644
--- a/tests/native_tests/py36_only_tests.hy
+++ b/tests/native_tests/py36_only_tests.hy
@@ -40,8 +40,8 @@
 
 (defn test-pep-487 []
   (defclass QuestBase []
-    [--init-subclass-- (fn [cls swallow &kwargs kwargs]
-                         (setv cls.swallow swallow))])
+    (defn --init-subclass-- [cls swallow &kwargs kwargs]
+                         (setv cls.swallow swallow)))
 
   (defclass Quest [QuestBase :swallow "african"])
   (assert (= (. (Quest) swallow) "african")))
diff --git a/tests/native_tests/with_decorator.hy b/tests/native_tests/with_decorator.hy
index 68dd1fc..fa6abc1 100644
--- a/tests/native_tests/with_decorator.hy
+++ b/tests/native_tests/with_decorator.hy
@@ -27,7 +27,7 @@
     cls)
   (with-decorator bardec
     (defclass cls []
-      [attr1 123]))
+      (setv attr1 123)))
   (assert (= cls.attr1 123))
   (assert (= cls.attr2 456)))
 
diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
index e98fb6d..5611177 100644
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -142,13 +142,13 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little
 
 (defclass C2 [C1]
   "class docstring"
-  [attr1 5  attr2 6]
-  (setv attr3 7))
+  (setv attr1 5)
+  (setv attr2 6))
 
 (import [contextlib [closing]])
 (setv closed [])
 (defclass Closeable []
-  [close (fn [self] (.append closed self.x))])
+  (defn close [self] (.append closed self.x)))
 (with [c1 (closing (Closeable)) c2 (closing (Closeable))]
   (setv c1.x "v1")
   (setv c2.x "v2"))
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
index 6428eef..69edc7e 100644
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -113,7 +113,7 @@ def assert_stuff(m):
 
     assert m.C2.__doc__ == "class docstring"
     assert issubclass(m.C2, m.C1)
-    assert (m.C2.attr1, m.C2.attr2, m.C2.attr3) == (5, 6, 7)
+    assert (m.C2.attr1, m.C2.attr2) == (5, 6)
 
     assert m.closed == ["v2", "v1"]
 

From 6bc9e842e17c45086f6d07d0a83c91cf326256c3 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 2 Jul 2019 12:10:01 -0400
Subject: [PATCH 172/223] Clean up whitespace

---
 hy/contrib/sequences.hy               | 87 +++++++++++++++------------
 tests/native_tests/defclass.hy        |  2 +-
 tests/native_tests/language.hy        | 18 +++---
 tests/native_tests/mathematics.hy     | 26 ++++----
 tests/native_tests/py36_only_tests.hy |  2 +-
 5 files changed, 72 insertions(+), 63 deletions(-)

diff --git a/hy/contrib/sequences.hy b/hy/contrib/sequences.hy
index b794b7a..01226c9 100644
--- a/hy/contrib/sequences.hy
+++ b/hy/contrib/sequences.hy
@@ -3,53 +3,60 @@
 ;; license. See the LICENSE.
 
 (defclass Sequence []
+
   (defn --init-- [self func]
-              "initialize a new sequence with a function to compute values"
-              (setv (. self func) func)
-              (setv (. self cache) [])
-              (setv (. self high-water) -1))
+    "initialize a new sequence with a function to compute values"
+    (setv (. self func) func)
+    (setv (. self cache) [])
+    (setv (. self high-water) -1))
+
   (defn --getitem-- [self n]
-                 "get nth item of sequence"
-                 (if (hasattr n "start")
-                   (gfor x (range n.start n.stop (or n.step 1))
-                         (get self x))
-                   (do (when (neg? n)
-                         ; Call (len) to force the whole
-                         ; sequence to be evaluated.
-                         (len self))
-                       (if (<= n (. self high-water))
-                         (get (. self cache) n)
-                         (do (while (< (. self high-water) n)
-                               (setv (. self high-water) (inc (. self high-water)))
-                               (.append (. self cache) (.func self (. self high-water))))
-                             (get self n))))))
+    "get nth item of sequence"
+    (if (hasattr n "start")
+    (gfor x (range n.start n.stop (or n.step 1))
+         (get self x))
+    (do (when (neg? n)
+         ; Call (len) to force the whole
+         ; sequence to be evaluated.
+         (len self))
+       (if (<= n (. self high-water))
+         (get (. self cache) n)
+         (do (while (< (. self high-water) n)
+               (setv (. self high-water) (inc (. self high-water)))
+               (.append (. self cache) (.func self (. self high-water))))
+             (get self n))))))
+
    (defn --iter-- [self]
-              "create iterator for this sequence"
-              (setv index 0)
-              (try (while True
-                     (yield (get self index))
-                     (setv index (inc index)))
-                   (except [IndexError]
-                     (return))))
+     "create iterator for this sequence"
+     (setv index 0)
+     (try (while True
+            (yield (get self index))
+            (setv index (inc index)))
+          (except [IndexError]
+            (return))))
+
    (defn --len-- [self]
-             "length of the sequence, dangerous for infinite sequences"
-             (setv index (. self high-water))
-             (try (while True
-                    (get self index)
-                    (setv index (inc index)))
-                  (except [IndexError]
-                    (len (. self cache)))))
+     "length of the sequence, dangerous for infinite sequences"
+     (setv index (. self high-water))
+     (try (while True
+            (get self index)
+            (setv index (inc index)))
+          (except [IndexError]
+            (len (. self cache)))))
+
    (setv max-items-in-repr 10)
+
    (defn --str-- [self]
-             "string representation of this sequence"
-             (setv items (list (take (inc self.max-items-in-repr) self)))
-             (.format (if (> (len items) self.max-items-in-repr)
-                        "[{0}, ...]"
-                        "[{0}]")
-                      (.join ", " (map str items))))
+     "string representation of this sequence"
+     (setv items (list (take (inc self.max-items-in-repr) self)))
+     (.format (if (> (len items) self.max-items-in-repr)
+                "[{0}, ...]"
+                "[{0}]")
+              (.join ", " (map str items))))
+
    (defn --repr-- [self]
-              "string representation of this sequence"
-              (.--str-- self)))
+     "string representation of this sequence"
+     (.--str-- self)))
 
 (defmacro seq [param &rest seq-code]
   `(Sequence (fn ~param (do ~@seq-code))))
diff --git a/tests/native_tests/defclass.hy b/tests/native_tests/defclass.hy
index 8fc76bc..e27a20e 100644
--- a/tests/native_tests/defclass.hy
+++ b/tests/native_tests/defclass.hy
@@ -37,7 +37,7 @@
   (defclass B []
     (setv x 42)
     (setv y (fn [self value]
-         (+ self.x value))))
+      (+ self.x value))))
   (assert (= B.x 42))
   (assert (= (.y (B) 5) 47))
   (setv b (B))
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index d7bf1da..e251840 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1773,20 +1773,22 @@ macros()
 
 (defn test-pep-3115 []
   (defclass member-table [dict]
-    (defn --init-- [self] (setv self.member-names []))
+    (defn --init-- [self]
+      (setv self.member-names []))
 
     (defn --setitem-- [self key value]
-                   (if (not-in key self)
-                       (.append self.member-names key))
-                   (dict.--setitem-- self key value)))
+      (if (not-in key self)
+          (.append self.member-names key))
+      (dict.--setitem-- self key value)))
 
   (defclass OrderedClass [type]
-    (setv --prepare-- (classmethod (fn [metacls name bases] (member-table))))
+    (setv --prepare-- (classmethod (fn [metacls name bases]
+      (member-table))))
 
     (defn --new-- [cls name bases classdict]
-               (setv result (type.--new-- cls name bases (dict classdict)))
-               (setv result.member-names classdict.member-names)
-               result))
+      (setv result (type.--new-- cls name bases (dict classdict)))
+      (setv result.member-names classdict.member-names)
+      result))
 
   (defclass MyClass [:metaclass OrderedClass]
     (defn method1 [self] (pass))
diff --git a/tests/native_tests/mathematics.hy b/tests/native_tests/mathematics.hy
index 23c9045..5069644 100644
--- a/tests/native_tests/mathematics.hy
+++ b/tests/native_tests/mathematics.hy
@@ -160,19 +160,19 @@
 
 (defclass HyTestMatrix [list]
   (defn --matmul-- [self other]
-     (setv n (len self)
-           m (len (. other [0]))
-           result [])
-     (for [i (range m)]
-       (setv result-row [])
-       (for [j (range n)]
-         (setv dot-product 0)
-         (for [k (range (len (. self [0])))]
-           (+= dot-product (* (. self [i] [k])
-                              (. other [k] [j]))))
-         (.append result-row dot-product))
-       (.append result result-row))
-     result))
+    (setv n (len self)
+          m (len (. other [0]))
+          result [])
+    (for [i (range m)]
+      (setv result-row [])
+      (for [j (range n)]
+        (setv dot-product 0)
+        (for [k (range (len (. self [0])))]
+          (+= dot-product (* (. self [i] [k])
+                             (. other [k] [j]))))
+        (.append result-row dot-product))
+      (.append result result-row))
+    result))
 
 (setv first-test-matrix (HyTestMatrix [[1 2 3]
                                        [4 5 6]
diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy
index 7f5908a..3ba0c4f 100644
--- a/tests/native_tests/py36_only_tests.hy
+++ b/tests/native_tests/py36_only_tests.hy
@@ -41,7 +41,7 @@
 (defn test-pep-487 []
   (defclass QuestBase []
     (defn --init-subclass-- [cls swallow &kwargs kwargs]
-                         (setv cls.swallow swallow)))
+      (setv cls.swallow swallow)))
 
   (defclass Quest [QuestBase :swallow "african"])
   (assert (= (. (Quest) swallow) "african")))

From c99360b294acbe96f8c1d710cf532869864c7b83 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 2 Jul 2019 12:04:15 -0400
Subject: [PATCH 173/223] Remove support for `defclass` attribute lists

---
 NEWS.rst       |  2 ++
 hy/compiler.py | 10 ++--------
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 00103d4..a0a00a9 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -6,6 +6,8 @@ Unreleased
 Removals
 ------------------------------
 * Python 2 is no longer supported.
+* Support for attribute lists in `defclass` has been removed. Use `setv`
+  and `defn` instead.
 
 0.17.0
 ==============================
diff --git a/hy/compiler.py b/hy/compiler.py
index 7638893..27ea786 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1475,10 +1475,9 @@ class HyASTCompiler(object):
 
     @special("defclass", [
         SYM,
-        maybe(brackets(many(FORM)) + maybe(STR) +
-              maybe(brackets(many(SYM + FORM))) + many(FORM))])
+        maybe(brackets(many(FORM)) + maybe(STR) + many(FORM))])
     def compile_class_expression(self, expr, root, name, rest):
-        base_list, docstring, attrs, body = rest or ([[]], None, None, [])
+        base_list, docstring, body = rest or ([[]], None, [])
 
         bases_expr, bases, keywords = (
             self._compile_collect(base_list[0], with_kwargs=True))
@@ -1488,11 +1487,6 @@ class HyASTCompiler(object):
         if docstring is not None:
             bodyr += self.compile(docstring).expr_as_stmt()
 
-        if attrs is not None:
-            bodyr += self.compile(self._rewire_init(HyExpression(
-                [HySymbol("setv")] +
-                [x for pair in attrs[0] for x in pair]).replace(attrs)))
-
         for e in body:
             e = self.compile(self._rewire_init(
                 macroexpand(e, self.module, self)))

From 8b101d12142580c692974c9b317ceaaff65efcd1 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 2 Jul 2019 12:16:06 -0400
Subject: [PATCH 174/223] Update documentation

---
 docs/contrib/hy_repr.rst |  4 ++--
 docs/language/api.rst    | 14 +++++++-------
 docs/tutorial.rst        |  6 +++---
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/contrib/hy_repr.rst b/docs/contrib/hy_repr.rst
index fae2625..4b0bf9a 100644
--- a/docs/contrib/hy_repr.rst
+++ b/docs/contrib/hy_repr.rst
@@ -67,8 +67,8 @@ your choice to the keyword argument ``:placeholder`` of
 .. code-block:: hy
 
    (defclass Container [object]
-     [__init__ (fn [self value]
-       (setv self.value value))])
+     (defn __init__ (fn [self value]
+       (setv self.value value))))
    (hy-repr-register Container :placeholder "HY THERE" (fn [x]
      (+ "(Container " (hy-repr x.value) ")")))
    (setv container (Container 5))
diff --git a/docs/language/api.rst b/docs/language/api.rst
index 6b243ba..ca967a5 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -430,16 +430,16 @@ Whereas ``setv`` creates an assignment statement, ``setx`` creates an assignment
 defclass
 --------
 
-New classes are declared with ``defclass``. It can take three optional parameters in the following order:
-a list defining (a) possible super class(es), a string (:term:`py:docstring`) and another list containing
-attributes of the new class along with their corresponding values.
+New classes are declared with ``defclass``. It can take optional parameters in the following order:
+a list defining (a) possible super class(es) and a string (:term:`py:docstring`).
 
 .. code-block:: clj
 
     (defclass class-name [super-class-1 super-class-2]
       "docstring"
-      [attribute1 value1
-       attribute2 value2]
+
+      (setv attribute1 value1)
+      (setv attribute2 value2)
 
       (defn method [self] (print "hello!")))
 
@@ -449,8 +449,8 @@ below:
 .. code-block:: clj
 
     => (defclass Cat []
-    ...  [age None
-    ...   colour "white"]
+    ...  (setv age None)
+    ...  (setv colour "white")
     ...
     ...  (defn speak [self] (print "Meow")))
 
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index cb2b272..e7dfa44 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -527,9 +527,9 @@ In Hy:
 .. code-block:: clj
 
   (defclass Customer [models.Model]
-    [name (models.CharField :max-length 255)
-     address (models.TextField)
-     notes (models.TextField)])
+     (setv name (models.CharField :max-length 255))
+     (setv address (models.TextField))
+     (setv notes (models.TextField)))
 
 Macros
 ======

From 0fcf570a3fb143c787b25b5c4e5691b3f05930d6 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 18 Jul 2019 10:43:01 -0400
Subject: [PATCH 175/223] Document `await`

---
 docs/language/api.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index ca967a5..fd246a0 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -224,6 +224,23 @@ Examples of usage:
 .. note:: ``assoc`` modifies the datastructure in place and returns ``None``.
 
 
+await
+-----
+
+``await`` creates an :ref:`await expression <py:await>`. It takes exactly one
+argument: the object to wait for.
+
+::
+
+    => (import asyncio)
+    => (defn/a main []
+    ...   (print "hello")
+    ...   (await (asyncio.sleep 1))
+    ...   (print "world"))
+    => (asyncio.run (main))
+    hello
+    world
+
 break
 -----
 

From 349da353d656bc6dc9a5ef39a1c292f006016137 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Jul 2019 15:40:01 -0400
Subject: [PATCH 176/223] Factor out compiler subs for constructing HyExprs

---
 hy/compiler.py | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 27ea786..23afbe2 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -324,6 +324,18 @@ def is_unpack(kind, x):
             and x[0] == "unpack-" + kind)
 
 
+def make_hy_model(outer, x, rest):
+   return outer(
+      [HySymbol(a) if type(a) is str else
+              a[0] if type(a) is list else a
+          for a in x] +
+      (rest or []))
+def mkexpr(*items, **kwargs):
+   return make_hy_model(HyExpression, items, kwargs.get('rest'))
+def mklist(*items, **kwargs):
+   return make_hy_model(HyList, items, kwargs.get('rest'))
+
+
 class HyASTCompiler(object):
     """A Hy-to-Python AST compiler"""
 
@@ -1364,19 +1376,17 @@ class HyASTCompiler(object):
             # We need to ensure the statements for the condition are
             # executed on every iteration. Rewrite the loop to use a
             # single anonymous variable as the condition.
-            def e(*x): return HyExpression(x)
-            s = HySymbol
-            cond_var = s(self.get_anon_var())
-            return self.compile(e(
-                s('do'),
-                e(s('setv'), cond_var, 1),
-                e(s('while'), cond_var,
+            cond_var = self.get_anon_var()
+            return self.compile(mkexpr(
+                'do',
+                mkexpr('setv', cond_var, 'True'),
+                mkexpr('while', cond_var,
                   # Cast the condition to a bool in case it's mutable and
                   # changes its truth value, but use (not (not ...)) instead of
                   # `bool` in case `bool` has been redefined.
-                  e(s('setv'), cond_var, e(s('not'), e(s('not'), cond))),
-                  e(s('if*'), cond_var, e(s('do'), *body)),
-                  *([e(s('else'), *else_expr)] if else_expr is not None else []))).replace(expr))  # noqa
+                  mkexpr('setv', cond_var, mkexpr('not', mkexpr('not', [cond]))),
+                  mkexpr('if*', cond_var, mkexpr('do', rest=body)),
+                  *([mkexpr('else', rest=else_expr)] if else_expr is not None else []))).replace(expr))  # noqa
 
         orel = Result()
         if else_expr is not None:

From 3afb4fdabebd06462613fd80de7d9a0267d2756c Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Jul 2019 15:40:45 -0400
Subject: [PATCH 177/223] Use `mkexpr` in HyASTCompiler.imports_as_stmts

---
 hy/compiler.py | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 23afbe2..bb1da22 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -402,21 +402,11 @@ class HyASTCompiler(object):
         ret = Result()
         for module, names in self.imports.items():
             if None in names:
-                e = HyExpression([
-                        HySymbol("import"),
-                        HySymbol(module),
-                    ]).replace(expr)
-                ret += self.compile(e)
+                ret += self.compile(mkexpr('import', module).replace(expr))
             names = sorted(name for name in names if name)
             if names:
-                e = HyExpression([
-                        HySymbol("import"),
-                        HyList([
-                            HySymbol(module),
-                            HyList([HySymbol(name) for name in names])
-                        ])
-                    ]).replace(expr)
-                ret += self.compile(e)
+                ret += self.compile(mkexpr('import',
+                    mklist(module, mklist(*names))))
         self.imports = defaultdict(set)
         return ret.stmts
 

From d99cf809866974eac317927f43431ef9028eacec Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 8 Jul 2019 15:41:13 -0400
Subject: [PATCH 178/223] Run statements in the second argument of `assert`

I've edited the test to use a list instead of a set because the order of evaluation probably ought to be guaranteed.
---
 NEWS.rst                       |  4 ++++
 hy/compiler.py                 | 21 ++++++++++++++++-----
 tests/native_tests/language.hy |  7 +++----
 3 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index a0a00a9..5b30c1a 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -9,6 +9,10 @@ Removals
 * Support for attribute lists in `defclass` has been removed. Use `setv`
   and `defn` instead.
 
+Bug Fixes
+------------------------------
+* Statements in the second argument of `assert` are now executed.
+
 0.17.0
 ==============================
 
diff --git a/hy/compiler.py b/hy/compiler.py
index bb1da22..7f81f6b 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -820,11 +820,22 @@ class HyASTCompiler(object):
 
     @special("assert", [FORM, maybe(FORM)])
     def compile_assert_expression(self, expr, root, test, msg):
-        ret = self.compile(test)
-        e = ret.force_expr
-        if msg is not None:
-            msg = self.compile(msg).force_expr
-        return ret + asty.Assert(expr, test=e, msg=msg)
+        if msg is None or type(msg) is HySymbol:
+            ret = self.compile(test)
+            return ret + asty.Assert(
+                expr,
+                test=ret.force_expr,
+                msg=(None if msg is None else self.compile(msg).force_expr))
+
+        # The `msg` part may involve statements, which we only
+        # want to be executed if the assertion fails. Rewrite the
+        # form to set `msg` to a variable.
+        msg_var = self.get_anon_var()
+        return self.compile(mkexpr(
+            'if*', mkexpr('and', '__debug__', mkexpr('not', [test])),
+                mkexpr('do',
+                    mkexpr('setv', msg_var, [msg]),
+                    mkexpr('assert', 'False', msg_var))).replace(expr))
 
     @special(["global", "nonlocal"], [oneplus(SYM)])
     def compile_global_or_nonlocal(self, expr, root, syms):
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index e251840..707ecf0 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1653,16 +1653,15 @@ macros()
    (= (identify-keywords 1 "bloo" :foo)
       ["other" "other" "keyword"])))
 
-#@(pytest.mark.xfail
 (defn test-assert-multistatements []
   ; https://github.com/hylang/hy/issues/1390
-  (setv s (set))
+  (setv l [])
   (defn f [x]
-    (.add s x)
+    (.append l x)
     False)
   (with [(pytest.raises AssertionError)]
     (assert (do (f 1) (f 2)) (do (f 3) (f 4))))
-  (assert (= s #{1 2 3 4}))))
+  (assert (= l [1 2 3 4])))
 
 (defn test-underscore_variables []
   ; https://github.com/hylang/hy/issues/1340

From 289f172d560b8575c43ba82ae52d6e4ae02fbdb1 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sun, 7 Jul 2019 18:53:22 -0500
Subject: [PATCH 179/223] Fix #1790: Rework statements in while condition

This avoids compiling them more than once while also applying some simplification.
---
 NEWS.rst                       |  1 +
 hy/compiler.py                 | 45 ++++++++++++++++++++++------------
 tests/native_tests/language.hy | 18 +++++++++++++-
 3 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 5b30c1a..7556193 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -12,6 +12,7 @@ Removals
 Bug Fixes
 ------------------------------
 * Statements in the second argument of `assert` are now executed.
+* Fixed the expression of a while loop that contains statements being compiled twice.
 
 0.17.0
 ==============================
diff --git a/hy/compiler.py b/hy/compiler.py
index 7f81f6b..648ff7b 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1373,33 +1373,46 @@ class HyASTCompiler(object):
     def compile_while_expression(self, expr, root, cond, body, else_expr):
         cond_compiled = self.compile(cond)
 
+        body = self._compile_branch(body)
+        body += body.expr_as_stmt()
+        body_stmts = body.stmts or [asty.Pass(expr)]
+
         if cond_compiled.stmts:
             # We need to ensure the statements for the condition are
             # executed on every iteration. Rewrite the loop to use a
-            # single anonymous variable as the condition.
-            cond_var = self.get_anon_var()
-            return self.compile(mkexpr(
-                'do',
-                mkexpr('setv', cond_var, 'True'),
-                mkexpr('while', cond_var,
-                  # Cast the condition to a bool in case it's mutable and
-                  # changes its truth value, but use (not (not ...)) instead of
-                  # `bool` in case `bool` has been redefined.
-                  mkexpr('setv', cond_var, mkexpr('not', mkexpr('not', [cond]))),
-                  mkexpr('if*', cond_var, mkexpr('do', rest=body)),
-                  *([mkexpr('else', rest=else_expr)] if else_expr is not None else []))).replace(expr))  # noqa
+            # single anonymous variable as the condition, i.e.:
+            #  anon_var = True
+            #  while anon_var:
+            #    condition stmts...
+            #    anon_var = condition expr
+            #    if anon_var:
+            #      while loop body
+            cond_var = asty.Name(cond, id=self.get_anon_var(), ctx=ast.Load())
+            def make_not(operand):
+                return asty.UnaryOp(cond, op=ast.Not(), operand=operand)
+
+            body_stmts = cond_compiled.stmts + [
+                asty.Assign(cond, targets=[self._storeize(cond, cond_var)],
+                            # Cast the condition to a bool in case it's mutable and
+                            # changes its truth value, but use (not (not ...)) instead of
+                            # `bool` in case `bool` has been redefined.
+                            value=make_not(make_not(cond_compiled.force_expr))),
+                asty.If(cond, test=cond_var, body=body_stmts, orelse=[]),
+            ]
+
+            cond_compiled = (Result()
+                + asty.Assign(cond, targets=[self._storeize(cond, cond_var)],
+                              value=asty.Name(cond, id="True", ctx=ast.Load()))
+                + cond_var)
 
         orel = Result()
         if else_expr is not None:
             orel = self._compile_branch(else_expr)
             orel += orel.expr_as_stmt()
 
-        body = self._compile_branch(body)
-        body += body.expr_as_stmt()
-
         ret = cond_compiled + asty.While(
             expr, test=cond_compiled.force_expr,
-            body=body.stmts or [asty.Pass(expr)],
+            body=body_stmts,
             orelse=orel.stmts)
 
         return ret
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 707ecf0..c525f7c 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -197,7 +197,23 @@
     (.append l 1)
     (len l))
   (while (!= (f) 4) (do))
-  (assert (= l [1 1 1 1])))
+  (assert (= l [1 1 1 1]))
+
+  ; only compile the condition once
+  ; https://github.com/hylang/hy/issues/1790
+  (global while-cond-var)
+  (setv while-cond-var 10)
+  (eval
+    '(do
+      (defmacro while-cond []
+        (global while-cond-var)
+        (assert (= while-cond-var 10))
+        (+= while-cond-var 1)
+        `(do
+          (setv x 3)
+          False))
+      (while (while-cond))
+      (assert (= x 3)))))
 
 (defn test-while-loop-else []
   (setv count 5)

From 6f3b6ca7358eb60651262e4fbda64eb57107b9a9 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Sun, 7 Jul 2019 19:28:12 -0500
Subject: [PATCH 180/223] Pre-Python 3.4 cleanup (including Python 2)

---
 hy/__main__.py             |  2 +-
 hy/importer.py             |  8 ++++++++
 setup.py                   |  8 +++-----
 tests/importer/test_pyc.py | 12 +++++++-----
 tests/test_hy2py.py        | 16 +++-------------
 5 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/hy/__main__.py b/hy/__main__.py
index 8f180b4..b6e8575 100644
--- a/hy/__main__.py
+++ b/hy/__main__.py
@@ -11,5 +11,5 @@ import sys
 
 if len(sys.argv) > 1:
     sys.argv.pop(0)
-    imp.load_source("__main__", sys.argv[0])
+    hy.importer._import_from_path('__main__', sys.argv[0])
     sys.exit(0)  # right?
diff --git a/hy/importer.py b/hy/importer.py
index f54803d..bd0f544 100644
--- a/hy/importer.py
+++ b/hy/importer.py
@@ -157,3 +157,11 @@ runpy = importlib.import_module('runpy')
 
 _runpy_get_code_from_file = runpy._get_code_from_file
 runpy._get_code_from_file = _get_code_from_file
+
+
+def _import_from_path(name, path):
+    """A helper function that imports a module from the given path."""
+    spec = importlib.util.spec_from_file_location(name, path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
diff --git a/setup.py b/setup.py
index bd6718e..9499601 100755
--- a/setup.py
+++ b/setup.py
@@ -39,8 +39,6 @@ install_requires = [
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 
-ver = sys.version_info[0]
-
 setup(
     name=PKG,
     version=__version__,
@@ -49,11 +47,11 @@ setup(
     entry_points={
         'console_scripts': [
             'hy = hy.cmdline:hy_main',
-            'hy%d = hy.cmdline:hy_main' % ver,
+            'hy3 = hy.cmdline:hy_main',
             'hyc = hy.cmdline:hyc_main',
-            'hyc%d = hy.cmdline:hyc_main' % ver,
+            'hyc3 = hy.cmdline:hyc_main',
             'hy2py = hy.cmdline:hy2py_main',
-            'hy2py%d = hy.cmdline:hy2py_main' % ver,
+            'hy2py3 = hy.cmdline:hy2py_main',
         ]
     },
     packages=find_packages(exclude=['tests*']),
diff --git a/tests/importer/test_pyc.py b/tests/importer/test_pyc.py
index 287e9e3..3d903a6 100644
--- a/tests/importer/test_pyc.py
+++ b/tests/importer/test_pyc.py
@@ -3,10 +3,11 @@
 # license. See the LICENSE.
 
 import os
-import imp
+import importlib.util
+import py_compile
 import tempfile
 
-import py_compile
+import hy.importer
 
 
 def test_pyc():
@@ -16,10 +17,11 @@ def test_pyc():
         f.flush()
 
         cfile = py_compile.compile(f.name)
-
         assert os.path.exists(cfile)
 
-        mod = imp.load_compiled('pyc', cfile)
-        os.remove(cfile)
+        try:
+            mod = hy.importer._import_from_path('pyc', cfile)
+        finally:
+            os.remove(cfile)
 
         assert mod.pyctest('Foo') == 'XFooY'
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
index 69edc7e..46ef638 100644
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -6,6 +6,7 @@
 import math, itertools
 from hy import mangle
 from hy._compat import PY36
+import hy.importer
 
 
 def test_direct_import():
@@ -19,7 +20,8 @@ def test_hy2py_import(tmpdir):
         ["hy2py", "tests/resources/pydemo.hy"]).decode("UTF-8")
     path = tmpdir.join("pydemo.py")
     path.write(python_code)
-    assert_stuff(import_from_path("pydemo", path))
+    # Note: explicit "str" is needed for 3.5.
+    assert_stuff(hy.importer._import_from_path("pydemo", str(path)))
 
 
 def assert_stuff(m):
@@ -116,15 +118,3 @@ def assert_stuff(m):
     assert (m.C2.attr1, m.C2.attr2) == (5, 6)
 
     assert m.closed == ["v2", "v1"]
-
-
-def import_from_path(name, path):
-    if PY36:
-        import importlib.util
-        spec = importlib.util.spec_from_file_location(name, path)
-        m = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(m)
-    else:
-        import imp
-        m = imp.load_source(name, str(path))
-    return m

From d32e460531cd4bcab40df7f68fece8db31ab3ae0 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 24 Jul 2019 11:21:11 -0400
Subject: [PATCH 181/223] Remove some exceptions in keyword parsing

---
 NEWS.rst                       |  2 ++
 hy/compiler.py                 |  6 +-----
 tests/native_tests/core.hy     |  4 ++--
 tests/native_tests/language.hy | 19 ++++++++++++-------
 4 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 7556193..d56dce5 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -8,6 +8,8 @@ Removals
 * Python 2 is no longer supported.
 * Support for attribute lists in `defclass` has been removed. Use `setv`
   and `defn` instead.
+* Literal keywords are no longer parsed differently in calls to functions
+  with certain names.
 
 Bug Fixes
 ------------------------------
diff --git a/hy/compiler.py b/hy/compiler.py
index 648ff7b..4387366 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1664,11 +1664,7 @@ class HyASTCompiler(object):
         if not func:
             func = self.compile(root)
 
-        # An exception for pulling together keyword args is if we're doing
-        # a typecheck, eg (type :foo)
-        with_kwargs = root not in (
-            "type", "HyKeyword", "keyword", "name", "keyword?", "identity")
-        args, ret, keywords = self._compile_collect(args, with_kwargs)
+        args, ret, keywords = self._compile_collect(args, with_kwargs=True)
 
         return func + ret + asty.Call(
             expr, func=func.expr, args=args, keywords=keywords)
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index eb1b1c3..b961f78 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -634,8 +634,8 @@ result['y in globals'] = 'y' in globals()")
   "NATIVE: testing the keyword? function"
   (assert (keyword? ':bar))
   (assert (keyword? ':baz))
-  (assert (keyword? :bar))
-  (assert (keyword? :baz))
+  (setv x :bar)
+  (assert (keyword? x))
   (assert (not (keyword? "foo")))
   (assert (not (keyword? ":foo")))
   (assert (not (keyword? 1)))
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index c525f7c..688f985 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -1104,7 +1104,8 @@
 
   (assert (= :foo :foo))
   (assert (= :foo ':foo))
-  (assert (is (type :foo) (type ':foo)))
+  (setv x :foo)
+  (assert (is (type x) (type ':foo)))
   (assert (= (get {:foo "bar"} :foo) "bar"))
   (assert (= (get {:bar "quux"} (get {:foo :bar} :foo)) "quux")))
 
@@ -1119,9 +1120,9 @@
 (defn test-empty-keyword []
   "NATIVE: test that the empty keyword is recognized"
   (assert (= : :))
-  (assert (keyword? :))
+  (assert (keyword? ':))
   (assert (!= : ":"))
-  (assert (= (name :) "")))
+  (assert (= (name ':) "")))
 
 
 (defn test-nested-if []
@@ -1198,12 +1199,15 @@
       5j 5.1j 2+1j 1.2+3.4j
       "" b""
       "apple bloom" b"apple bloom" "⚘" b"\x00"
-      :mykeyword
       [] #{} {}
       [1 2 3] #{1 2 3} {"a" 1 "b" 2}]]
     (assert (= (eval `(identity ~x)) x))
     (assert (= (eval x) x)))
 
+  (setv kw :mykeyword)
+  (assert (= (get (eval `[~kw]) 0) kw))
+  (assert (= (eval kw) kw))
+
   ; Tuples wrap to HyLists, not HyExpressions.
   (assert (= (eval (,)) []))
   (assert (= (eval (, 1 2 3)) [1 2 3]))
@@ -1632,7 +1636,8 @@ macros()
   (assert (= (keyword 'foo) :foo))
   (assert (= (keyword 'foo-bar) :foo-bar))
   (assert (= (keyword 1) :1))
-  (assert (= (keyword :foo_bar) :foo-bar)))
+  (setv x :foo_bar)
+  (assert (= (keyword x) :foo-bar)))
 
 (defn test-name-conversion []
   "NATIVE: Test name conversion"
@@ -1644,8 +1649,8 @@ macros()
   (assert (= (name 'foo_bar) "foo-bar"))
   (assert (= (name 1) "1"))
   (assert (= (name 1.0) "1.0"))
-  (assert (= (name :foo) "foo"))
-  (assert (= (name :foo_bar) "foo-bar"))
+  (assert (= (name ':foo) "foo"))
+  (assert (= (name ':foo_bar) "foo-bar"))
   (assert (= (name test-name-conversion) "test-name-conversion")))
 
 (defn test-keywords []

From 6fb6eefd6b4e27eb4a9e9f394674c1439ac803be Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 24 Jul 2019 13:32:05 -0400
Subject: [PATCH 182/223] Make augmented assignment operators variadic

---
 NEWS.rst                        |  5 ++++
 hy/compiler.py                  | 43 +++++++++++++++++++--------------
 tests/native_tests/operators.hy | 40 ++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 18 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index d56dce5..476177c 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -11,6 +11,11 @@ Removals
 * Literal keywords are no longer parsed differently in calls to functions
   with certain names.
 
+New Features
+------------------------------
+* All augmented assignment operators (except `%=` and `^=`) now allow
+  more than two arguments.
+
 Bug Fixes
 ------------------------------
 * Statements in the second argument of `assert` are now executed.
diff --git a/hy/compiler.py b/hy/compiler.py
index 4387366..6d09afa 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1265,19 +1265,21 @@ class HyASTCompiler(object):
         return ret + asty.Compare(
             expr, left=exprs[0], ops=ops, comparators=exprs[1:])
 
-    m_ops = {"+": ast.Add,
-             "/": ast.Div,
-             "//": ast.FloorDiv,
-             "*": ast.Mult,
-             "-": ast.Sub,
-             "%": ast.Mod,
-             "**": ast.Pow,
-             "<<": ast.LShift,
-             ">>": ast.RShift,
-             "|": ast.BitOr,
-             "^": ast.BitXor,
-             "&": ast.BitAnd,
-             "@": ast.MatMult}
+    # The second element of each tuple below is an aggregation operator
+    # that's used for augmented assignment with three or more arguments.
+    m_ops = {"+": (ast.Add, "+"),
+             "/": (ast.Div, "*"),
+             "//": (ast.FloorDiv, "*"),
+             "*": (ast.Mult, "*"),
+             "-": (ast.Sub, "+"),
+             "%": (ast.Mod, None),
+             "**": (ast.Pow, "**"),
+             "<<": (ast.LShift, "+"),
+             ">>": (ast.RShift, "+"),
+             "|": (ast.BitOr, "|"),
+             "^": (ast.BitXor, None),
+             "&": (ast.BitAnd, "&"),
+             "@": (ast.MatMult, "@")}
 
     @special(["+", "*", "|"], [many(FORM)])
     @special(["-", "/", "&", "@"], [oneplus(FORM)])
@@ -1302,7 +1304,7 @@ class HyASTCompiler(object):
                 # Return the argument unchanged.
                 return self.compile(args[0])
 
-        op = self.m_ops[root]
+        op = self.m_ops[root][0]
         right_associative = root == "**"
         ret = self.compile(args[-1 if right_associative else 0])
         for child in args[-2 if right_associative else 1 ::
@@ -1318,11 +1320,16 @@ class HyASTCompiler(object):
 
     a_ops = {x + "=": v for x, v in m_ops.items()}
 
-    @special(list(a_ops.keys()), [FORM, FORM])
-    def compile_augassign_expression(self, expr, root, target, value):
-        op = self.a_ops[root]
+    @special([x for x, (_, v) in a_ops.items() if v is not None], [FORM, oneplus(FORM)])
+    @special([x for x, (_, v) in a_ops.items() if v is None], [FORM, times(1, 1, FORM)])
+    def compile_augassign_expression(self, expr, root, target, values):
+        if len(values) > 1:
+            return self.compile(mkexpr(root, [target],
+                mkexpr(self.a_ops[root][1], rest=values)).replace(expr))
+
+        op = self.a_ops[root][0]
         target = self._storeize(target, self.compile(target))
-        ret = self.compile(value)
+        ret = self.compile(values[0])
         return ret + asty.AugAssign(
             expr, target=target, value=ret.force_expr, op=op())
 
diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy
index 7c9b18b..4445138 100644
--- a/tests/native_tests/operators.hy
+++ b/tests/native_tests/operators.hy
@@ -303,3 +303,43 @@
   (assert (= (f "hello" 1) "e"))
   (assert (= (f [[1 2 3] [4 5 6] [7 8 9]] 1 2) 6))
   (assert (= (f {"x" {"y" {"z" 12}}} "x" "y" "z") 12)))
+
+
+(defn test-augassign []
+    (setv b 2  c 3  d 4)
+    (defmacro same-as [expr1 expr2 expected-value]
+      `(do
+        (setv a 4)
+        ~expr1
+        (setv expr1-value a)
+        (setv a 4)
+        ~expr2
+        (assert (= expr1-value a ~expected-value))))
+    (same-as (+= a b c d) (+= a (+ b c d)) 13)
+    (same-as (-= a b c d) (-= a (+ b c d)) -5)
+    (same-as (*= a b c d) (*= a (* b c d)) 96)
+    (same-as (**= a b c) (**= a (** b c)) 65,536)
+    (same-as (/= a b c d) (/= a (* b c d)) (/ 1 6))
+    (same-as (//= a b c d) (//= a (* b c d)) 0)
+    (same-as (<<= a b c d) (<<= a (+ b c d)) 0b10_00000_00000)
+    (same-as (>>= a b c d) (>>= a (+ b c d)) 0)
+    (same-as (&= a b c d) (&= a (& b c d)) 0)
+    (same-as (|= a b c d) (|= a (| b c d)) 0b111)
+
+    (defclass C [object]
+      (defn __init__ [self content] (setv self.content content))
+      (defn __matmul__ [self other] (C (+ self.content other.content))))
+    (setv  a (C "a")  b (C "b")  c (C "c")  d (C "d"))
+    (@= a b c d)
+    (assert (= a.content "abcd"))
+    (setv a (C "a"))
+    (@= a (@ b c d))
+    (assert (= a.content "abcd"))
+
+    (setv a 15)
+    (%= a 9)
+    (assert (= a 6))
+
+    (setv a 0b1100)
+    (^= a 0b1010)
+    (assert (= a 0b0110)))

From a9c6ab639190d5fb1c0e03e6c2dee65cfefa6407 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 24 Jul 2019 13:28:17 -0400
Subject: [PATCH 183/223] Remove native_tests/mathematics

It should now be fully redundant with native_tests/operators.
---
 tests/native_tests/mathematics.hy | 199 ------------------------------
 1 file changed, 199 deletions(-)
 delete mode 100644 tests/native_tests/mathematics.hy

diff --git a/tests/native_tests/mathematics.hy b/tests/native_tests/mathematics.hy
deleted file mode 100644
index 5069644..0000000
--- a/tests/native_tests/mathematics.hy
+++ /dev/null
@@ -1,199 +0,0 @@
-;; Copyright 2019 the authors.
-;; This file is part of Hy, which is free software licensed under the Expat
-;; license. See the LICENSE.
-
-(setv square (fn [x]
-               (* x x)))
-
-
-(setv test_basic_math (fn []
-                        "NATIVE: Test basic math."
-                        (assert (= (+ 2 2) 4))))
-
-(setv test_mult (fn []
-                  "NATIVE: Test multiplication."
-                  (assert (= 4 (square 2)))
-                  (assert (= 8 (* 8)))
-                  (assert (= 1 (*)))))
-
-
-(setv test_sub (fn []
-                 "NATIVE: Test subtraction"
-                 (assert (= 4 (- 8 4)))
-                 (assert (= -8 (- 8)))))
-
-
-(setv test_add (fn []
-                 "NATIVE: Test addition"
-                 (assert (= 4 (+ 1 1 1 1)))
-                 (assert (= 8 (+ 8)))
-                 (assert (= 0 (+)))))
-
-
-(defn test-add-unary []
-  "NATIVE: test that unary + calls __pos__"
-
-  (defclass X [object]
-    (defn __pos__ [self] "called __pos__"))
-  (assert (= (+ (X)) "called __pos__"))
-
-  ; Make sure the shadowed version works, too.
-  (setv f +)
-  (assert (= (f (X)) "called __pos__")))
-
-
-(setv test_div (fn []
-                 "NATIVE: Test division"
-                 (assert (= 25 (/ 100 2 2)))
-                                ; Commented out until float constants get implemented
-                                ; (assert (= 0.5 (/ 1 2)))
-                 (assert (= 1 (* 2 (/ 1 2))))))
-
-(setv test_int_div (fn []
-                     "NATIVE: Test integer division"
-                     (assert (= 25 (// 101 2 2)))))
-
-(defn test-modulo []
-  "NATIVE: test mod"
-  (assert (= (% 10 2) 0)))
-
-(defn test-pow []
-  "NATIVE: test pow"
-  (assert (= (** 10 2) 100)))
-
-(defn test-lshift []
-  "NATIVE: test lshift"
-  (assert (= (<< 1 2) 4)))
-
-(defn test-rshift []
-  "NATIVE: test lshift"
-  (assert (= (>> 8 1) 4)))
-
-(defn test-bitor []
-  "NATIVE: test lshift"
-  (assert (= (| 1 2) 3)))
-
-(defn test-bitxor []
-  "NATIVE: test xor"
-  (assert (= (^ 1 2) 3)))
-
-(defn test-bitand []
-  "NATIVE: test lshift"
-  (assert (= (& 1 2) 0)))
-
-(defn test-augassign-add []
-  "NATIVE: test augassign add"
-  (setv x 1)
-  (+= x 41)
-  (assert (= x 42)))
-
-(defn test-augassign-sub []
-  "NATIVE: test augassign sub"
-  (setv x 1)
-  (-= x 41)
-  (assert (= x -40)))
-
-(defn test-augassign-mult []
-  "NATIVE: test augassign mult"
-  (setv x 1)
-  (*= x 41)
-  (assert (= x 41)))
-
-(defn test-augassign-div []
-  "NATIVE: test augassign div"
-  (setv x 42)
-  (/= x 2)
-  (assert (= x 21)))
-
-(defn test-augassign-floordiv []
-  "NATIVE: test augassign floordiv"
-  (setv x 42)
-  (//= x 2)
-  (assert (= x 21)))
-
-(defn test-augassign-mod []
-  "NATIVE: test augassign mod"
-  (setv x 42)
-  (%= x 2)
-  (assert (= x 0)))
-
-(defn test-augassign-pow []
-  "NATIVE: test augassign pow"
-  (setv x 2)
-  (**= x 3)
-  (assert (= x 8)))
-
-(defn test-augassign-lshift []
-  "NATIVE: test augassign lshift"
-  (setv x 2)
-  (<<= x 2)
-  (assert (= x 8)))
-
-(defn test-augassign-rshift []
-  "NATIVE: test augassign rshift"
-  (setv x 8)
-  (>>= x 1)
-  (assert (= x 4)))
-
-(defn test-augassign-bitand []
-  "NATIVE: test augassign bitand"
-  (setv x 8)
-  (&= x 1)
-  (assert (= x 0)))
-
-(defn test-augassign-bitor []
-  "NATIVE: test augassign bitand"
-  (setv x 0)
-  (|= x 2)
-  (assert (= x 2)))
-
-(defn test-augassign-bitxor []
-  "NATIVE: test augassign bitand"
-  (setv x 1)
-  (^= x 1)
-  (assert (= x 0)))
-
-(defn overflow-int-to-long []
-  "NATIVE: test if int does not raise an overflow exception"
-  (assert (integer? (+ 1 1000000000000000000000000))))
-
-
-(defclass HyTestMatrix [list]
-  (defn --matmul-- [self other]
-    (setv n (len self)
-          m (len (. other [0]))
-          result [])
-    (for [i (range m)]
-      (setv result-row [])
-      (for [j (range n)]
-        (setv dot-product 0)
-        (for [k (range (len (. self [0])))]
-          (+= dot-product (* (. self [i] [k])
-                             (. other [k] [j]))))
-        (.append result-row dot-product))
-      (.append result result-row))
-    result))
-
-(setv first-test-matrix (HyTestMatrix [[1 2 3]
-                                       [4 5 6]
-                                       [7 8 9]]))
-
-(setv second-test-matrix (HyTestMatrix [[2 0 0]
-                                        [0 2 0]
-                                        [0 0 2]]))
-
-(setv product-of-test-matrices (HyTestMatrix [[ 2  4  6]
-                                              [ 8 10 12]
-                                              [14 16 18]]))
-
-(defn test-matmul []
-  "NATIVE: test matrix multiplication"
-  (assert (= (@ first-test-matrix second-test-matrix)
-             product-of-test-matrices)))
-
-(defn test-augassign-matmul []
-  "NATIVE: test augmented-assignment matrix multiplication"
-  (setv matrix first-test-matrix
-        matmul-attempt (try (@= matrix second-test-matrix)
-                              (except [e [Exception]] e)))
-  (assert (= product-of-test-matrices matrix)))

From 84fbf04f8478b9fdd0fee4c9166f7df1c8099195 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 30 Jul 2019 15:14:33 -0400
Subject: [PATCH 184/223] Fix AST representation of format strings

---
 NEWS.rst                  | 1 +
 hy/compiler.py            | 8 ++++++--
 tests/resources/pydemo.hy | 4 ++++
 tests/test_hy2py.py       | 3 +++
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 476177c..36a681e 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -20,6 +20,7 @@ Bug Fixes
 ------------------------------
 * Statements in the second argument of `assert` are now executed.
 * Fixed the expression of a while loop that contains statements being compiled twice.
+* `hy2py` can now handle format strings.
 
 0.17.0
 ==============================
diff --git a/hy/compiler.py b/hy/compiler.py
index 6d09afa..34a9dd5 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1784,7 +1784,7 @@ class HyASTCompiler(object):
                item = item[2:].lstrip()
 
            # Look for a format specifier.
-           format_spec = asty.Str(string, s = "")
+           format_spec = None
            if item.startswith(':'):
                if allow_recursion:
                    ret += self._format_string(string,
@@ -1793,6 +1793,8 @@ class HyASTCompiler(object):
                    format_spec = ret.force_expr
                else:
                    format_spec = asty.Str(string, s=item[1:])
+                   if PY36:
+                       format_spec = asty.JoinedStr(string, values=[format_spec])
            elif item:
                raise self._syntax_error(string,
                    "f-string: trailing junk in field")
@@ -1818,7 +1820,9 @@ class HyASTCompiler(object):
                            ('!' + conversion if conversion else '') +
                            ':{}}'),
                        attr = 'format', ctx = ast.Load()),
-                   args = [ret.force_expr, format_spec],
+                   args = [
+                       ret.force_expr,
+                       format_spec or asty.Str(string, s = "")],
                    keywords = [], starargs = None, kwargs = None))
 
         return ret + (
diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
index 5611177..a23e3c2 100644
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -49,6 +49,10 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little
 (setv condexpr (if "" "x" "y"))
 (setv mylambda (fn [x] (+ x "z")))
 
+(setv fstring1 f"hello {(+ 1 1)} world")
+(setv p "xyzzy")
+(setv fstring2 f"a{p !r :9}")
+
 (setv augassign 103)
 (//= augassign 4)
 
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
index 46ef638..d19e4f6 100644
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -79,6 +79,9 @@ def assert_stuff(m):
     assert type(m.mylambda) is type(lambda x: x + "z")
     assert m.mylambda("a") == "az"
 
+    assert m.fstring1 == "hello 2 world"
+    assert m.fstring2 == "a'xyzzy'  "
+
     assert m.augassign == 25
 
     assert m.delstatement == ["a", "c", "d", "e"]

From ad74a92e2d937254731de64d92e5b942f678b62c Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 16 Aug 2019 19:03:34 -0400
Subject: [PATCH 185/223] Avoid a crash when we can't access a history file

---
 NEWS.rst        | 1 +
 hy/completer.py | 7 +++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 36a681e..72e3581 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -21,6 +21,7 @@ Bug Fixes
 * Statements in the second argument of `assert` are now executed.
 * Fixed the expression of a while loop that contains statements being compiled twice.
 * `hy2py` can now handle format strings.
+* Fixed crashes from inaccessible history files.
 
 0.17.0
 ==============================
diff --git a/hy/completer.py b/hy/completer.py
index b1969c1..34308ba 100644
--- a/hy/completer.py
+++ b/hy/completer.py
@@ -123,7 +123,7 @@ def completion(completer=None):
         try:
             readline.read_history_file(history)
         except IOError:
-            open(history, 'a').close()
+            pass
 
         readline.parse_and_bind(readline_bind)
 
@@ -131,4 +131,7 @@ def completion(completer=None):
         yield
     finally:
         if docomplete:
-            readline.write_history_file(history)
+            try:
+                readline.write_history_file(history)
+            except IOError:
+                pass

From 6929973d0df1221b8de34698c094e353528a425d Mon Sep 17 00:00:00 2001
From: lsusr <root@lsusr.com>
Date: Sat, 17 Aug 2019 00:39:04 -0700
Subject: [PATCH 186/223] Fixed broken link to Graphviz

---
 docs/contrib/profile.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/contrib/profile.rst b/docs/contrib/profile.rst
index 157f415..eca03ee 100644
--- a/docs/contrib/profile.rst
+++ b/docs/contrib/profile.rst
@@ -17,7 +17,7 @@ profile/calls
 --------------
 
 ``profile/calls`` allows you to create a call graph visualization.
-**Note:** You must have `Graphviz <http://www.graphviz.org/Home.php>`_
+**Note:** You must have `Graphviz <http://www.graphviz.org/>`_
 installed for this to work.
 
 

From bc524daee8e270fa1030a2445d7569d773488160 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 17:16:43 -0400
Subject: [PATCH 187/223] Remove dead code

---
 tests/native_tests/contrib/walk.hy | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index 2e59cfe..854b412 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -31,7 +31,6 @@
   (assert (= acc [walk-form])))
 
 (defn test-walk-iterators []
-  (setv acc [])
   (assert (= (walk (fn [x] (* 2 x)) (fn [x] x)
                    (drop 1 [1 [2 [3 [4]]]]))
              [[2 [3 [4]] 2 [3 [4]]]])))

From 88c0f92810d062680ab26e7493f8eb2310a3530b Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Thu, 1 Aug 2019 13:35:24 -0400
Subject: [PATCH 188/223] Clean up string handling in _compile_assign

---
 hy/compiler.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 34a9dd5..7b48822 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1345,17 +1345,16 @@ class HyASTCompiler(object):
 
     def _compile_assign(self, root, name, result):
 
-        str_name = "%s" % name
-        if str_name in ("None", "True", "False"):
+        if name in [HySymbol(x) for x in ("None", "True", "False")]:
             raise self._syntax_error(name,
-                "Can't assign to `%s'" % str_name)
+                "Can't assign to `{}'".format(name))
 
         result = self.compile(result)
         ld_name = self.compile(name)
 
         if isinstance(ld_name.expr, ast.Call):
             raise self._syntax_error(name,
-                "Can't assign to a callable: `%s'" % str_name)
+                "Can't assign to a callable: `{}'".format(name))
 
         if (result.temp_variables
                 and isinstance(name, HySymbol)

From 6cced31738b359e5b10a2067e50a4099cbdd4324 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 14:46:34 -0400
Subject: [PATCH 189/223] Rewrite `cond`

---
 hy/core/macros.hy | 34 ++++++++++++----------------------
 1 file changed, 12 insertions(+), 22 deletions(-)

diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 1021c76..dc5e035 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -87,29 +87,19 @@ Shorthand for nested with/a* loops:
 
 The result in the bracket may be omitted, in which case the condition is also
 used as the result."
-  (if (empty? branches)
-    None
-    (do
-     (setv branches (iter branches))
-     (setv branch (next branches))
-     (defn check-branch [branch]
-       "check `cond` branch for validity, return the corresponding `if` expr"
-       (if (not (= (type branch) HyList))
-         (macro-error branch "cond branches need to be a list"))
-       (if (< (len branch) 2)
-         (do
-           (setv g (gensym))
-           `(if (do (setv ~g ~(first branch)) ~g) ~g))
-         `(if ~(first branch) (do ~@(cut branch 1)))))
+  (or branches
+    (return))
 
-     (setv root (check-branch branch))
-     (setv latest-branch root)
-
-     (for [branch branches]
-       (setv cur-branch (check-branch branch))
-       (.append latest-branch cur-branch)
-       (setv latest-branch cur-branch))
-     root)))
+  `(if ~@(reduce + (gfor
+    branch branches
+    (if
+      (not (and (is (type branch) hy.HyList) branch))
+        (macro-error branch "each cond branch needs to be a nonempty list")
+      (= (len branch) 1) (do
+        (setv g (gensym))
+        [`(do (setv ~g ~(first branch)) ~g) g])
+      True
+        [(first branch) `(do ~@(cut branch 1))])))))
 
 
 (defmacro -> [head &rest args]

From 9ddb3b1031bd8be37847b3a44cde2e96a2f85c83 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 14:53:57 -0400
Subject: [PATCH 190/223] Rewrite `_with`

---
 hy/core/macros.hy                  | 18 ++++++++----------
 tests/native_tests/contrib/walk.hy | 12 ++++++------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index dc5e035..90ddc22 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -48,16 +48,14 @@ be associated in pairs."
 
 
 (defn _with [node args body]
-  (if (not (empty? args))
-    (do
-     (if (>= (len args) 2)
-       (do
-        (setv p1 (.pop args 0)
-              p2 (.pop args 0)
-              primary [p1 p2])
-        `(~node [~@primary] ~(_with node args body)))
-       `(~node [~@args] ~@body)))
-    `(do ~@body)))
+  (if
+    (not args)
+      `(do ~@body)
+    (<= (len args) 2)
+      `(~node [~@args] ~@body)
+    True (do
+      (setv [p1 p2 #* args] args)
+      `(~node [~p1 ~p2] ~(_with node args body)))))
 
 
 (defmacro with [args &rest body]
diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index 854b412..d23e4ae 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -43,19 +43,19 @@
   (assert (= (macroexpand-all '(foo-walk))
              42))
   (assert (= (macroexpand-all '(with [a 1]))
-             '(with* [a 1] (do))))
+             '(with* [a 1])))
   (assert (= (macroexpand-all '(with [a 1 b 2 c 3] (for [d c] foo)))
-             '(with* [a 1] (with* [b 2] (with* [c 3] (do (for [d c] foo)))))))
+             '(with* [a 1] (with* [b 2] (with* [c 3] (for [d c] foo))))))
   (assert (= (macroexpand-all '(with [a 1]
                                  '(with [b 2])
                                  `(with [c 3]
                                     ~(with [d 4])
                                     ~@[(with [e 5])])))
              '(with* [a 1]
-                (do '(with [b 2])
-                    `(with [c 3]
-                       ~(with* [d 4] (do))
-                       ~@[(with* [e 5] (do))])))))
+                '(with [b 2])
+                `(with [c 3]
+                   ~(with* [d 4])
+                   ~@[(with* [e 5])]))))
 
   (defmacro require-macro []
     `(do

From 95ad5a01c829bfa856fef63d78262a77df444f66 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 17:05:28 -0400
Subject: [PATCH 191/223] Avoid mutating HyLists in hy.contrib

---
 hy/contrib/multi.hy | 10 +++-------
 hy/contrib/walk.hy  |  4 ++--
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/hy/contrib/multi.hy b/hy/contrib/multi.hy
index 1a19fc9..08016e2 100644
--- a/hy/contrib/multi.hy
+++ b/hy/contrib/multi.hy
@@ -84,13 +84,9 @@
      (setv comment (HyString))
      (if (= (type (first bodies)) HyString)
        (setv [comment bodies] (head-tail bodies)))
-     (setv ret `(do))
-     (.append ret '(import [hy.contrib.multi [MultiDispatch]]))
-     (for [body bodies]
-       (setv [let-binds body] (head-tail body))
-       (.append ret 
-                `(with-decorator MultiDispatch (defn ~name ~let-binds ~comment ~@body))))
-     ret)
+     (+ '(do (import [hy.contrib.multi [MultiDispatch]])) (lfor
+       [let-binds #* body] bodies
+       `(with-decorator MultiDispatch (defn ~name ~let-binds ~comment ~@body)))))
     (do
      (setv [lambda-list body] (head-tail bodies))
      `(setv ~name (fn* ~lambda-list ~@body)))))
diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index d30f29e..40f0c62 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -89,7 +89,7 @@ splits a fn argument list into sections based on &-headers.
 returns an OrderedDict mapping headers to sublists.
 Arguments without a header are under None.
 "
-  (setv headers '[&optional &rest &kwonly &kwargs]
+  (setv headers ['&optional '&rest '&kwonly '&kwargs]
         sections (OrderedDict [(, None [])])
         header None)
   (for [arg form]
@@ -169,7 +169,7 @@ Arguments without a header are under None.
                                                 #{})))))
   (defn handle-args-list [self]
     (setv protected #{}
-          argslist `[])
+          argslist [])
     (for [[header section] (-> self (.tail) first lambda-list .items)]
       (if header (.append argslist header))
       (cond [(in header [None '&rest '&kwargs])

From 4a40ff3d7e9cbfbfa319758d8bf24ab6c19e8cd7 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 17:06:26 -0400
Subject: [PATCH 192/223] Check for HySequence in hy.contrib.walk

---
 hy/contrib/walk.hy | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index 40f0c62..def9049 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -4,6 +4,7 @@
 ;; license. See the LICENSE.
 
 (import [hy [HyExpression HyDict]]
+        [hy.models [HySequence]]
         [functools [partial]]
         [importlib [import-module]]
         [collections [OrderedDict]]
@@ -17,9 +18,7 @@
   (cond
    [(instance? HyExpression form)
     (outer (HyExpression (map inner form)))]
-   [(instance? HyDict form)
-    (HyDict (outer (HyExpression (map inner form))))]
-   [(list? form)
+   [(or (instance? HySequence form) (list? form))
     ((type form) (outer (HyExpression (map inner form))))]
    [(coll? form)
     (walk inner outer (list form))]

From dce0e10f3f073b621d8c93de76306d57b22f7000 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 17:08:27 -0400
Subject: [PATCH 193/223] Use `nonlocal` instead of a singleton list

---
 hy/contrib/walk.hy | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy
index def9049..691645e 100644
--- a/hy/contrib/walk.hy
+++ b/hy/contrib/walk.hy
@@ -45,22 +45,24 @@
   (setv module (or (and module-name
                         (import-module module-name))
                    (calling-module))
-        quote-level [0]
+        quote-level 0
         ast-compiler (HyASTCompiler module))  ; TODO: make nonlocal after dropping Python2
   (defn traverse [form]
     (walk expand identity form))
   (defn expand [form]
+    (nonlocal quote-level)
     ;; manages quote levels
     (defn +quote [&optional [x 1]]
+      (nonlocal quote-level)
       (setv head (first form))
-      (+= (get quote-level 0) x)
-      (when (neg? (get quote-level 0))
+      (+= quote-level x)
+      (when (neg? quote-level)
         (raise (TypeError "unquote outside of quasiquote")))
       (setv res (traverse (cut form 1)))
-      (-= (get quote-level 0) x)
+      (-= quote-level x)
       `(~head ~@res))
     (if (call? form)
-        (cond [(get quote-level 0)
+        (cond [quote-level
                (cond [(in (first form) '[unquote unquote-splice])
                       (+quote -1)]
                      [(= (first form) 'quasiquote) (+quote)]

From 52c0e4e221dbbf48c8f68f6630db42f27204af69 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 17:09:13 -0400
Subject: [PATCH 194/223] Add explicit checks for HyList

---
 hy/contrib/hy_repr.hy | 2 +-
 hy/core/macros.hy     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hy/contrib/hy_repr.hy b/hy/contrib/hy_repr.hy
index 3baaebb..a3c7631 100644
--- a/hy/contrib/hy_repr.hy
+++ b/hy/contrib/hy_repr.hy
@@ -144,7 +144,7 @@
     (hy-repr (dict x)))))
 
 (for [[types fmt] (partition [
-    list "[...]"
+    [list HyList] "[...]"
     [set HySet] "#{...}"
     frozenset "(frozenset #{...})"
     dict-keys "(dict-keys [...])"
diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index 90ddc22..e1d5d48 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -201,7 +201,7 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
   (defn extract-o!-sym [arg]
     (cond [(and (symbol? arg) (.startswith arg "o!"))
            arg]
-          [(and (list? arg) (.startswith (first arg) "o!"))
+          [(and (instance? HyList arg) (.startswith (first arg) "o!"))
            (first arg)]))
   (setv os (list (filter identity (map extract-o!-sym args)))
         gs (lfor s os (HySymbol (+ "g!" (cut s 2)))))

From 8576d00ce880b2079139e5bfaa760571b200d534 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 14:48:06 -0400
Subject: [PATCH 195/223] Implement HySequence with tuples instead of lists

---
 hy/models.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hy/models.py b/hy/models.py
index fbced02..56160df 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -243,7 +243,7 @@ class HyComplex(HyObject, complex):
 _wrappers[complex] = HyComplex
 
 
-class HySequence(HyObject, list):
+class HySequence(HyObject, tuple):
     """
     An abstract type for sequence-like models to inherit from.
     """
@@ -256,7 +256,8 @@ class HySequence(HyObject, list):
         return self
 
     def __add__(self, other):
-        return self.__class__(super(HySequence, self).__add__(other))
+        return self.__class__(super(HySequence, self).__add__(
+            tuple(other) if isinstance(other, list) else other))
 
     def __getslice__(self, start, end):
         return self.__class__(super(HySequence, self).__getslice__(start, end))

From cc8948d9b92f3030106241c24a22dd0239b19aea Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 17:13:40 -0400
Subject: [PATCH 196/223] Return plain lists from HyDict.keys, .values

---
 hy/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hy/models.py b/hy/models.py
index 56160df..55d0b79 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -327,10 +327,10 @@ class HyDict(HySequence):
                 return '' + g("HyDict()")
 
     def keys(self):
-        return self[0::2]
+        return list(self[0::2])
 
     def values(self):
-        return self[1::2]
+        return list(self[1::2])
 
     def items(self):
         return list(zip(self.keys(), self.values()))

From e6aac2308addb023e624739d45123d6cbd8ee101 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 2 Aug 2019 17:17:24 -0400
Subject: [PATCH 197/223] Update tests for tuple HySequences

---
 tests/native_tests/contrib/walk.hy | 8 ++++----
 tests/native_tests/quote.hy        | 2 +-
 tests/test_models.py               | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/native_tests/contrib/walk.hy b/tests/native_tests/contrib/walk.hy
index d23e4ae..08922bf 100644
--- a/tests/native_tests/contrib/walk.hy
+++ b/tests/native_tests/contrib/walk.hy
@@ -21,10 +21,10 @@
              walk-form)))
 
 (defn test-walk []
-  (setv acc '())
-  (assert (= (walk (partial collector acc) identity walk-form)
+  (setv acc [])
+  (assert (= (list (walk (partial collector acc) identity walk-form))
              [None None]))
-  (assert (= acc walk-form))
+  (assert (= acc (list walk-form)))
   (setv acc [])
   (assert (= (walk identity (partial collector acc) walk-form)
              None))
@@ -129,7 +129,7 @@
                '(foo `(bar a ~a ~"x"))))
     (assert (= `(foo ~@[a])
                '(foo "x")))
-    (assert (= `(foo `(bar [a] ~@[a] ~@~[a 'a `a] ~~@[a]))
+    (assert (= `(foo `(bar [a] ~@[a] ~@~(HyList [a 'a `a]) ~~@[a]))
                '(foo `(bar [a] ~@[a] ~@["x" a a] ~"x"))))))
 
 (defn test-let-except []
diff --git a/tests/native_tests/quote.hy b/tests/native_tests/quote.hy
index 84371f0..9560bcc 100644
--- a/tests/native_tests/quote.hy
+++ b/tests/native_tests/quote.hy
@@ -9,7 +9,7 @@
   "NATIVE: test for quoting functionality"
   (setv q (quote (a b c)))
   (assert (= (len q) 3))
-  (assert (= q [(quote a) (quote b) (quote c)])))
+  (assert (= q (HyExpression [(quote a) (quote b) (quote c)]))))
 
 
 (defn test-basic-quoting []
diff --git a/tests/test_models.py b/tests/test_models.py
index b9e6a90..52d4518 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -56,8 +56,8 @@ def test_list_add():
     a = HyList([1, 2, 3])
     b = HyList([3, 4, 5])
     c = a + b
-    assert c == [1, 2, 3, 3, 4, 5]
-    assert c.__class__ == HyList
+    assert c == HyList([1, 2, 3, 3, 4, 5])
+    assert type(c) is HyList
 
 
 def test_list_slice():
@@ -91,7 +91,7 @@ hyset = HySet([3, 1, 2, 2])
 
 
 def test_set():
-    assert hyset == [3, 1, 2, 2]
+    assert list(hyset) == [3, 1, 2, 2]
 
 
 def test_number_model_copy():

From 7aaece3725c675ee1d359ce9b44d2e391bb19f93 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sat, 3 Aug 2019 09:26:32 -0400
Subject: [PATCH 198/223] Use #* assignments instead of `head-tail`

---
 hy/contrib/multi.hy | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/hy/contrib/multi.hy b/hy/contrib/multi.hy
index 08016e2..c84adf9 100644
--- a/hy/contrib/multi.hy
+++ b/hy/contrib/multi.hy
@@ -70,9 +70,6 @@
        (with-decorator (method-decorator ~name)
          (defn ~name ~params ~@body))))
 
-(defn head-tail [l]
-  (, (get l 0) (cut l 1)))
-
 (defmacro defn [name &rest bodies]
   (setv arity-overloaded? (fn [bodies]
                             (if (isinstance (first bodies) HyString)
@@ -83,10 +80,10 @@
     (do
      (setv comment (HyString))
      (if (= (type (first bodies)) HyString)
-       (setv [comment bodies] (head-tail bodies)))
+       (setv [comment #* bodies] bodies))
      (+ '(do (import [hy.contrib.multi [MultiDispatch]])) (lfor
        [let-binds #* body] bodies
        `(with-decorator MultiDispatch (defn ~name ~let-binds ~comment ~@body)))))
     (do
-     (setv [lambda-list body] (head-tail bodies))
+     (setv [lambda-list #* body] bodies)
      `(setv ~name (fn* ~lambda-list ~@body)))))

From e5461f171c0c4e4f10123d6af75b16c2d15aaf0c Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Sun, 18 Aug 2019 09:45:40 -0400
Subject: [PATCH 199/223] Update NEWS and documentation

---
 NEWS.rst                    | 6 ++++++
 docs/language/internals.rst | 9 ++++-----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 36a681e..6e5c8ff 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -16,6 +16,12 @@ New Features
 * All augmented assignment operators (except `%=` and `^=`) now allow
   more than two arguments.
 
+Other Breaking Changes
+------------------------------
+* ``HySequence`` is now a subclass of ``tuple`` instead of ``list``.
+  Thus, a ``HyList`` will never be equal to a ``list``, and you can't
+  use ``.append``, ``.pop``, etc. on an expression or list.
+
 Bug Fixes
 ------------------------------
 * Statements in the second argument of `assert` are now executed.
diff --git a/docs/language/internals.rst b/docs/language/internals.rst
index a89b356..2fb29bc 100644
--- a/docs/language/internals.rst
+++ b/docs/language/internals.rst
@@ -60,6 +60,10 @@ Adding a HySequence to another iterable object reuses the class of the
 left-hand-side object, a useful behavior when you want to concatenate Hy
 objects in a macro, for instance.
 
+HySequences are (mostly) immutable: you can't add, modify, or remove
+elements. You can still append to a variable containing a HySequence with
+``+=`` and otherwise construct new HySequences out of old ones.
+
 
 .. _hylist:
 
@@ -90,11 +94,6 @@ HyDict
 ``hy.models.HyDict`` inherits :ref:`HySequence` for curly-bracketed
 ``{}`` expressions, which compile down to a Python dictionary literal.
 
-The decision of using a list instead of a dict as the base class for
-``HyDict`` allows easier manipulation of dicts in macros, with the added
-benefit of allowing compound expressions as dict keys (as, for instance,
-the :ref:`HyExpression` Python class isn't hashable).
-
 Atomic Models
 -------------
 

From 627455a3366a6ee95d8ef4abe9d36d26c2b8a8c1 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 13 Aug 2019 16:54:45 -0400
Subject: [PATCH 200/223] Add some documentation anchors

---
 docs/contrib/walk.rst     |  2 ++
 docs/hacking.rst          |  2 ++
 docs/language/api.rst     | 40 +++++++++++++++++++++++++++++++++++++++
 docs/language/interop.rst |  2 ++
 docs/language/syntax.rst  |  2 ++
 5 files changed, 48 insertions(+)

diff --git a/docs/contrib/walk.rst b/docs/contrib/walk.rst
index 2e36234..e6d333e 100644
--- a/docs/contrib/walk.rst
+++ b/docs/contrib/walk.rst
@@ -206,6 +206,8 @@ Recursively performs all possible macroexpansions in form, using the ``require``
 Macros
 ======
 
+.. _let:
+
 let
 ---
 
diff --git a/docs/hacking.rst b/docs/hacking.rst
index da964b5..064159a 100644
--- a/docs/hacking.rst
+++ b/docs/hacking.rst
@@ -1,3 +1,5 @@
+.. _hacking:
+
 ===============
  Hacking on Hy
 ===============
diff --git a/docs/language/api.rst b/docs/language/api.rst
index fd246a0..b357c48 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -1,3 +1,5 @@
+.. _special-forms:
+
 =================
 Built-Ins
 =================
@@ -279,6 +281,8 @@ This is completely discarded and doesn't expand to anything, not even ``None``.
    Hy
 
 
+.. _cond:
+
 cond
 ----
 
@@ -338,6 +342,8 @@ is only called on every other value in the list.
       (side-effect2 x))
 
 
+.. _do:
+
 do
 ----------
 
@@ -400,6 +406,8 @@ the second of which becomes each value.
     {0: 0, 1: 10, 2: 20, 3: 30, 4: 40}
 
 
+.. _setv:
+
 setv
 ----
 
@@ -444,6 +452,8 @@ Whereas ``setv`` creates an assignment statement, ``setx`` creates an assignment
     3 is greater than 0
 
 
+.. _defclass:
+
 defclass
 --------
 
@@ -914,6 +924,9 @@ raising an exception.
     => (first [])
     None
 
+
+.. _for:
+
 for
 ---
 
@@ -992,6 +1005,8 @@ written without accidental variable name clashes.
 
    Section :ref:`using-gensym`
 
+.. _get:
+
 get
 ---
 
@@ -1022,6 +1037,8 @@ successive elements in a nested structure. Example usage:
           index that is out of bounds.
 
 
+.. _gfor:
+
 gfor
 ----
 
@@ -1063,6 +1080,8 @@ keyword, the second function would have raised a ``NameError``.
     (set-a 5)
     (print-a)
 
+.. _if:
+
 if / if* / if-not
 -----------------
 
@@ -1188,6 +1207,8 @@ that ``import`` can be used.
     (import [sys [*]])
 
 
+.. _fn:
+
 fn
 -----------
 
@@ -1253,6 +1274,8 @@ last
     6
 
 
+.. _lfor:
+
 lfor
 ----
 
@@ -1396,6 +1419,8 @@ print
 .. note:: ``print`` always returns ``None``.
 
 
+.. _quasiquote:
+
 quasiquote
 ----------
 
@@ -1414,6 +1439,8 @@ using ``unquote`` (``~``). The evaluated form can also be spliced using
     ; equivalent to '(foo bar baz)
 
 
+.. _quote:
+
 quote
 -----
 
@@ -1431,6 +1458,8 @@ alternatively be written using the apostrophe (``'``) symbol.
     Hello World
 
 
+.. _require:
+
 require
 -------
 
@@ -1571,6 +1600,8 @@ sfor
 equivalent to ``(set (lfor CLAUSES VALUE))``. See `lfor`_.
 
 
+.. _cut:
+
 cut
 -----
 
@@ -1677,6 +1708,8 @@ the given conditional is ``False``. The following shows the expansion of this ma
       (do statement))
 
 
+.. _unpack-iterable:
+
 unpack-iterable, unpack-mapping
 -------------------------------
 
@@ -1717,6 +1750,8 @@ more than once in one expression (:pep:`3132`, :pep:`448`).
     [1, 2, 3, 4]
 
 
+.. _unquote:
+
 unquote
 -------
 
@@ -1792,6 +1827,8 @@ following shows the expansion of the macro.
     (if conditional (do statement))
 
 
+.. _while:
+
 while
 -----
 
@@ -1851,6 +1888,9 @@ prints
     In condition
     At end of outer loop
 
+
+.. _with:
+
 with
 ----
 
diff --git a/docs/language/interop.rst b/docs/language/interop.rst
index 6276659..cb009b4 100644
--- a/docs/language/interop.rst
+++ b/docs/language/interop.rst
@@ -1,3 +1,5 @@
+.. _interop:
+
 =====================
 Hy <-> Python interop
 =====================
diff --git a/docs/language/syntax.rst b/docs/language/syntax.rst
index 16ed838..2e2e60a 100644
--- a/docs/language/syntax.rst
+++ b/docs/language/syntax.rst
@@ -1,3 +1,5 @@
+.. _syntax:
+
 ==============
 Syntax
 ==============

From 1e77f38d10eff6cc707b8c2721d87d8d24c3b914 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 13 Aug 2019 16:55:21 -0400
Subject: [PATCH 201/223] Expand the documentation of `setv`

---
 docs/language/api.rst | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index b357c48..c4f877b 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -424,20 +424,19 @@ For example:
     => (counter [1 2 3 4 5 2 3] 2)
     2
 
-They can be used to assign multiple variables at once:
+You can provide more than one target–value pair, and the assignments will be made in order::
 
-.. code-block:: hy
+    (setv  x 1  y x  x 2)
+    (print x y)  ; => 2 1
 
-    => (setv a 1 b 2)
-    (1L, 2L)
-    => a
-    1L
-    => b
-    2L
-    =>
+You can perform parallel assignments or unpack the source value with square brackets and :ref:`unpack-iterable`::
 
+    (setv duo ["tim" "eric"])
+    (setv [guy1 guy2] duo)
+    (print guy1 guy2)  ; => tim eric
 
-``setv`` always returns ``None``.
+    (setv [letter1 letter2 #* others] "abcdefg")
+    (print letter1 letter2 others)   ; => a b ['c', 'd', 'e', 'f', 'g']
 
 
 setx

From 6c93fc6ff1c69ca8e7afd02302996216319cd9c6 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 19 Aug 2019 14:00:53 -0400
Subject: [PATCH 202/223] Overhaul introductory documentation

- Removed links to non-updated code and badges.
- Compressed `quickstart.rst` into a few sentences at the very start of the docs.
- Added a "Why Hy?" chapter discussing Hy's features and comparing Hy to Python and other Lisps.
- Rewrote the tutorial to be more accessible to non-Python programmers and to be greater in breadth but lesser in depth.
- Cut down on the self-congratulatory manic tone and exclamation points, while keeping the jokes I liked best.
---
 README.md           |  39 +--
 docs/index.rst      |  29 +-
 docs/quickstart.rst |  54 ---
 docs/tutorial.rst   | 809 +++++++++++++-------------------------------
 docs/whyhy.rst      | 142 ++++++++
 5 files changed, 396 insertions(+), 677 deletions(-)
 delete mode 100644 docs/quickstart.rst
 create mode 100644 docs/whyhy.rst

diff --git a/README.md b/README.md
index 21aeab4..68180d9 100644
--- a/README.md
+++ b/README.md
@@ -1,38 +1,22 @@
 Hy
 ==
 
-[![Build Status](https://img.shields.io/travis/hylang/hy/master.svg)](https://travis-ci.org/hylang/hy)
 [![Version](https://img.shields.io/pypi/v/hy.svg)](https://pypi.python.org/pypi/hy)
 
 <a href="https://xkcd.com/224/"><img title="We lost the documentation on quantum mechanics. You'll have to decode the regexes yourself." alt="XKCD #224" src="https://raw.github.com/hylang/shyte/18f6925e08684b0e1f52b2cc2c803989cd62cd91/imgs/xkcd.png"></a>
 
-Lisp and Python should love each other. Let's make it happen. [Try it](http://try-hy.appspot.com/).
+Lisp and Python should love each other. Let's make it happen.
 
-Hylarious Hacks
----------------
+Hy is a Lisp dialect that's embedded in Python. Since Hy transforms its Lisp
+code into Python abstract syntax tree (AST) objects, you have the whole
+beautiful world of Python at your fingertips, in Lisp form.
 
-* [Django + Lisp](https://github.com/paultag/djlisp/tree/master/djlisp)
-* [Python `sh` Fun](https://twitter.com/paultag/status/314925996442796032)
-* [Hy IRC Bot](https://github.com/hylang/hygdrop)
-* [miniKanren in Hy](https://github.com/algernon/adderall)
+To install the latest stable release of Hy, just use the command `pip3 install
+--user hy`. Then you can start an interactive read-eval-print loop (REPL) with
+the command `hy`, or run a Hy program with `hy myprogram.hy`.
 
-OK, so, why?
-------------
-
-Well. Python is awesome. So awesome, that we have so many tools to alter the
-language in a *core* way, but we never use them.
-
-Why?
-
-Well, I wrote Hy to help people realize one thing about Python:
-
-It's really awesome.
-
-Oh, and lisps are neat.
-
-![Cuddles the Hacker](https://i.imgur.com/QbPMXTN.png)
-
-(fan art from the one and only [doctormo](http://doctormo.deviantart.com/art/Cuddles-the-Hacker-372184766))
+* [Why Hy?](http://docs.hylang.org/en/master/whyhy.html)
+* [Tutorial](http://docs.hylang.org/en/master/tutorial.html)
 
 Project
 -------
@@ -41,10 +25,13 @@ Project
 * Documentation:
   * stable, for use with the latest stable release: http://hylang.org/
   * master, for use with the latest revision on GitHub: http://docs.hylang.org/en/master
-* Quickstart: http://hylang.org/en/stable/quickstart.html
 * Bug reports: We have no bugs! Your bugs are your own! (https://github.com/hylang/hy/issues)
 * License: MIT (Expat)
 * [Hacking on Hy](http://docs.hylang.org/en/master/hacking.html)
   * [Contributor Guidelines](http://docs.hylang.org/en/master/hacking.html#contributor-guidelines)
   * [Code of Conduct](http://docs.hylang.org/en/master/hacking.html#contributor-code-of-conduct)
 * IRC: Join #hy on [freenode](https://webchat.freenode.net/)
+
+![Cuddles the Hacker](https://i.imgur.com/QbPMXTN.png)
+
+(fan art from the one and only [doctormo](http://doctormo.deviantart.com/art/Cuddles-the-Hacker-372184766))
diff --git a/docs/index.rst b/docs/index.rst
index 4a6cf41..6089770 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,36 +1,27 @@
-Welcome to Hy's documentation!
-==============================
+The Hy Manual
+=============
 
 .. image:: _static/hy-logo-small.png
    :alt: Hy
    :align: left
 
-:Try Hy: https://try-hy.appspot.com
 :PyPI: https://pypi.python.org/pypi/hy
 :Source: https://github.com/hylang/hy
 :List: `hylang-discuss <https://groups.google.com/forum/#!forum/hylang-discuss>`_
-:IRC: ``#hy`` on Freenode
-:Build status:
-    .. image:: https://secure.travis-ci.org/hylang/hy.png
-        :alt: Travis CI
-        :target: http://travis-ci.org/hylang/hy
+:IRC: irc://chat.freenode.net/hy
 
-Hy is a wonderful dialect of Lisp that's embedded in Python.
+Hy is a Lisp dialect that's embedded in Python. Since Hy transforms its Lisp
+code into Python abstract syntax tree (AST) objects, you have the whole
+beautiful world of Python at your fingertips, in Lisp form.
 
-Since Hy transforms its Lisp code into the Python Abstract Syntax
-Tree, you have the whole beautiful world of Python at your fingertips,
-in Lisp form!
-
-
-Documentation Index
-===================
-
-Contents:
+To install the latest stable release of Hy, just use the command ``pip3 install
+--user hy``. Then you can start an interactive read-eval-print loop (REPL) with
+the command ``hy``, or run a Hy program with ``hy myprogram.hy``.
 
 .. toctree::
    :maxdepth: 3
 
-   quickstart
+   whyhy
    tutorial
    style-guide
    language/index
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
deleted file mode 100644
index ca00c73..0000000
--- a/docs/quickstart.rst
+++ /dev/null
@@ -1,54 +0,0 @@
-==========
-Quickstart
-==========
-
-.. image:: _static/cuddles-transparent-small.png
-   :alt: Karen Rustard's Cuddles
-
-(Thanks to Karen Rustad for Cuddles!)
-
-
-**HOW TO GET HY REAL FAST**:
-
-1. Create a `Virtual Python Environment
-   <https://pypi.python.org/pypi/virtualenv>`_.
-2. Activate your Virtual Python Environment.
-3. Install `hy from GitHub <https://github.com/hylang/hy>`_ with ``$ pip install git+https://github.com/hylang/hy.git``.
-4. Start a REPL with ``hy``.
-5. Type stuff in the REPL::
-
-       => (print "Hy!")
-       Hy!
-       => (defn salutationsnm [name] (print (+ "Hy " name "!")))
-       => (salutationsnm "YourName")
-       Hy YourName!
-
-       etc
-
-6. Hit CTRL-D when you're done.
-7. If you're familiar with Python, start the REPL using ``hy --spy`` to check what happens inside::
-
-       => (+ "Hyllo " "World" "!")
-       'Hyllo ' + 'World' + '!'
-
-       'Hyllo World!'
-
-*OMG! That's amazing! I want to write a Hy program.*
-
-8. Open up an elite programming editor and type::
-
-       #! /usr/bin/env hy
-       (print "I was going to code in Python syntax, but then I got Hy.")
-
-9. Save as ``awesome.hy``.
-10. Make it executable::
-
-        chmod +x awesome.hy
-
-11. And run your first Hy program::
-
-        ./awesome.hy
-
-12. Take a deep breath so as to not hyperventilate.
-13. Smile villainously and sneak off to your hydeaway and do
-    unspeakable things.
diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index e7dfa44..566417a 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -2,272 +2,174 @@
 Tutorial
 ========
 
-.. TODO
-..
-..  - How do I index into arrays or dictionaries?
-..  - Blow your mind with macros!
-..  - Where's my banana???
+.. image:: _static/cuddles-transparent-small.png
+   :alt: Karen Rustard's Cuddles
 
-Welcome to the Hy tutorial!
+This chapter provides a quick introduction to Hy. It assumes a basic background
+in programming, but no specific prior knowledge of Python or Lisp.
 
-In a nutshell, Hy is a Lisp dialect, but one that converts its
-structure into Python ... literally a conversion into Python's abstract
-syntax tree!  (Or to put it in more crude terms, Hy is lisp-stick on a
-Python!)
+Lisp-stick on a Python
+======================
 
-This is pretty cool because it means Hy is several things:
+Let's start with the classic::
 
- - A Lisp that feels very Pythonic
- - For Lispers, a great way to use Lisp's crazy powers but in the wide
-   world of Python's libraries (why yes, you now can write a Django
-   application in Lisp!)
- - For Pythonistas, a great way to start exploring Lisp, from the
-   comfort of Python!
- - For everyone: a pleasant language that has a lot of neat ideas!
+    (print "Hy, world!")
 
+This program calls the :func:`print` function, which, like all of Python's
+:ref:`built-in functions <py:built-in-funcs>`, is available in Hy.
 
-Basic intro to Lisp for Pythonistas
-===================================
+All of Python's :ref:`binary and unary operators <py:expressions>` are
+available, too, although ``==`` is spelled ``=`` in deference to Lisp
+tradition. Here's how we'd use the addition operator ``+``::
 
-Okay, maybe you've never used Lisp before, but you've used Python!
+    (+ 1 3)
 
-A "hello world" program in Hy is actually super simple. Let's try it:
+This code returns ``4``. It's equivalent to ``1 + 3`` in Python and many other
+languages. Languages in the `Lisp
+<https://en.wikipedia.org/wiki/Lisp_(programming_language)>`_ family, including
+Hy, use a prefix syntax: ``+``, just like ``print`` or ``sqrt``, appears before
+all of its arguments. The call is delimited by parentheses, but the opening
+parenthesis appears before the operator being called instead of after it, so
+instead of ``sqrt(2)``, we write ``(sqrt 2)``. Multiple arguments, such as the
+two integers in ``(+ 1 3)``, are separated by whitespace. Many operators,
+including ``+``, allow more than two arguments: ``(+ 1 2 3)`` is equivalent to
+``1 + 2 + 3``.
 
-.. code-block:: clj
+Here's a more complex example::
 
-   (print "hello world")
+    (- (* (+ 1 3 88) 2) 8)
 
-See?  Easy!  As you may have guessed, this is the same as the Python
-version of::
+This code returns ``176``. Why? We can see the infix equivalent with the
+command ``echo "(- (* (+ 1 3 88) 2) 8)" | hy2py``, which returns the Python
+code corresponding to the given Hy code, or by passing the ``--spy`` option to
+Hy when starting the REPL, which shows the Python equivalent of each input line
+before the result. The infix equivalent in this case is:
 
-  print("hello world")
+.. code-block:: python
 
-To add up some super simple math, we could do:
+    ((1 + 3 + 88) * 2) - 8
 
-.. code-block:: clj
+To evaluate this infix expression, you'd of course evaluate the innermost
+parenthesized expression first and work your way outwards. The same goes for
+Lisp. Here's what we'd get by evaluating the above Hy code one step at a time::
 
-   (+ 1 3)
+    (- (* (+ 1 3 88) 2) 8)
+    (- (* 92 2) 8)
+    (- 184 8)
+    176
 
-Which would return 4 and would be the equivalent of:
+The basic unit of Lisp syntax, which is similar to a C or Python expression, is
+the **form**. ``92``, ``*``, and ``(* 92 2)`` are all forms. A Lisp program
+consists of a sequence of forms nested within forms. Forms are typically
+separated from each other by whitespace, but some forms, such as string
+literals (``"Hy, world!"``), can contain whitespace themselves. An
+**expression** is a form enclosed in parentheses; its first child form, called
+the **head**, determines what the expression does, and should generally be a
+function, macro, or special operator. Functions are the most ordinary sort of
+head, whereas macros (described in more detail below) are functions executed at
+compile-time instead and return code to be executed at run-time. Special
+operators are one of :ref:`a fixed set of names <special-forms>` that are
+hard-coded into the compiler, and used to implement everything else.
 
-.. code-block:: clj
+Comments start with a ``;`` character and continue till the end of the line. A
+comment is functionally equivalent to whitespace. ::
 
-   1 + 3
+    (print (** 2 64))   ; Max 64-bit unsigned integer value
 
-What you'll notice is that the first item in the list is the function
-being called and the rest of the arguments are the arguments being
-passed in.  In fact, in Hy (as with most Lisps) we can pass in
-multiple arguments to the plus operator:
+Although ``#`` isn't a comment character in Hy, a Hy program can begin with a
+`shebang line <https://en.wikipedia.org/wiki/Shebang_(Unix)>`_, which Hy itself
+will ignore::
 
-.. code-block:: clj
+   #!/usr/bin/env hy
+   (print "Make me executable, and run me!")
 
-   (+ 1 3 55)
+Literals
+========
 
-Which would return 59.
+Hy has :ref:`literal syntax <syntax>` for all of the same data types that
+Python does. Here's an example of Hy code for each type and the Python
+equivalent.
 
-Maybe you've heard of Lisp before but don't know much about it.  Lisp
-isn't as hard as you might think, and Hy inherits from Python, so Hy
-is a great way to start learning Lisp.  The main thing that's obvious
-about Lisp is that there's a lot of parentheses.  This might seem
-confusing at first, but it isn't so hard.  Let's look at some simple
-math that's wrapped in a bunch of parentheses that we could enter into
-the Hy interpreter:
+==============  ================  =================
+Hy              Python            Type
+==============  ================  =================
+``1``           ``1``             :class:`int`
+``1.2``         ``1.2``           :class:`float`
+``4j``          ``4j``            :class:`complex`
+``True``        ``True``          :class:`bool`
+``None``        ``None``          :class:`NoneType`
+``"hy"``        ``'hy'``          :class:`str`
+``b"hy"``       ``b'hy'``         :class:`bytes`
+``(, 1 2 3)``   ``(1, 2, 3)``     :class:`tuple`
+``[1 2 3]``     ``[1, 2, 3]``     :class:`list`
+``#{1 2 3}``    ``{1, 2, 3}``     :class:`set`
+``{1 2  3 4}``  ``{1: 2, 3: 4}``  :class:`dict`
+==============  ================  =================
 
-.. code-block:: clj
+In addition, Hy has a Clojure-style literal syntax for
+:class:`fractions.Fraction`: ``1/3`` is equivalent to ``fractions.Fraction(1,
+3)``.
 
-   (setv result (- (/ (+ 1 3 88) 2) 8))
-
-This would return 38.0  But why?  Well, we could look at the equivalent
-expression in python::
-
-  result = ((1 + 3 + 88) / 2) - 8
-
-If you were to try to figure out how the above were to work in python,
-you'd of course figure out the results by solving each inner
-parenthesis.  That's the same basic idea in Hy.  Let's try this
-exercise first in Python::
-
-  result = ((1 + 3 + 88) / 2) - 8
-  # simplified to...
-  result = (92 / 2) - 8
-  # simplified to...
-  result = 46.0 - 8
-  # simplified to...
-  result = 38.0
-
-Now let's try the same thing in Hy:
-
-.. code-block:: clj
-
-   (setv result (- (/ (+ 1 3 88) 2) 8))
-   ; simplified to...
-   (setv result (- (/ 92 2) 8))
-   ; simplified to...
-   (setv result (- 46.0 8))
-   ; simplified to...
-   (setv result 38.0)
-
-As you probably guessed, this last expression with ``setv`` means to
-assign the variable "result" to 38.0.
-
-See?  Not too hard!
-
-This is the basic premise of Lisp. Lisp stands for "list
-processing"; this means that the structure of the program is
-actually lists of lists.  (If you're familiar with Python lists,
-imagine the entire same structure as above but with square brackets
-instead, and you'll be able to see the structure above as both a
-program and a data structure.)  This is easier to understand with more
-examples, so let's write a simple Python program, test it, and then
-show the equivalent Hy program::
-
-  def simple_conversation():
-      print("Hello!  I'd like to get to know you.  Tell me about yourself!")
-      name = input("What is your name? ")
-      age = input("What is your age? ")
-      print("Hello " + name + "!  I see you are " + age + " years old.")
-
-  simple_conversation()
-
-If we ran this program, it might go like::
-
-  Hello!  I'd like to get to know you.  Tell me about yourself!
-  What is your name? Gary
-  What is your age? 38
-  Hello Gary!  I see you are 38 years old.
-
-Now let's look at the equivalent Hy program:
-
-.. code-block:: clj
-
-   (defn simple-conversation []
-      (print "Hello!  I'd like to get to know you.  Tell me about yourself!")
-      (setv name (input "What is your name? "))
-      (setv age (input "What is your age? "))
-      (print (+ "Hello " name "!  I see you are "
-                 age " years old.")))
-
-   (simple-conversation)
-
-If you look at the above program, as long as you remember that the
-first element in each list of the program is the function (or
-macro... we'll get to those later) being called and that the rest are
-the arguments, it's pretty easy to figure out what this all means.
-(As you probably also guessed, ``defn`` is the Hy method of defining
-methods.)
-
-Still, lots of people find this confusing at first because there's so
-many parentheses, but there are plenty of things that can help make
-this easier: keep indentation nice and use an editor with parenthesis
-matching (this will help you figure out what each parenthesis pairs up
-with) and things will start to feel comfortable.
-
-There are some advantages to having a code structure that's actually a
-very simple data structure as the core of Lisp is based on.  For one
-thing, it means that your programs are easy to parse and that the
-entire actual structure of the program is very clearly exposed to you.
-(There's an extra step in Hy where the structure you see is converted
-to Python's own representations ... in "purer" Lisps such as Common
-Lisp or Emacs Lisp, the data structure you see in the code and the
-data structure that is executed is much more literally close.)
-
-Another implication of this is macros: if a program's structure is a
-simple data structure, that means you can write code that can write
-code very easily, meaning that implementing entirely new language
-features can be very fast.  Previous to Hy, this wasn't very possible
-for Python programmers ... now you too can make use of macros'
-incredible power (just be careful to not aim them footward)!
-
-
-Hy is a Lisp-flavored Python
-============================
-
-Hy converts to Python's own abstract syntax tree, so you'll soon start
-to find that all the familiar power of python is at your fingertips.
-
-You have full access to Python's data types and standard library in
-Hy.  Let's experiment with this in the hy interpreter::
+The Hy REPL prints output in Python syntax by default::
 
   => [1 2 3]
   [1, 2, 3]
-  => {"dog" "bark"
-  ... "cat" "meow"}
-  {'dog': 'bark', 'cat': 'meow'}
-  => (, 1 2 3)
-  (1, 2, 3)
-  => #{3 1 2}
-  {1, 2, 3}
-  => 1/2
-  Fraction(1, 2)
 
-Notice the last two lines: Hy has a fraction literal like Clojure.
-
-If you start Hy like this (a shell alias might be helpful)::
+But if you start Hy like this (a shell alias might be helpful)::
 
   $ hy --repl-output-fn=hy.contrib.hy-repr.hy-repr
 
-the interactive mode will use :ref:`hy-repr-fn` instead of Python's
-native ``repr`` function to print out values, so you'll see values in
-Hy syntax rather than Python syntax::
+the interactive mode will use :ref:`hy-repr-fn` instead of Python's native
+``repr`` function to print out values, so you'll see values in Hy syntax::
 
   => [1 2 3]
   [1 2 3]
-  => {"dog" "bark"
-  ... "cat" "meow"}
-  {"dog" "bark" "cat" "meow"}
 
-If you are familiar with other Lisps, you may be interested that Hy
-supports the Common Lisp method of quoting:
 
-.. code-block:: clj
+Basic operations
+================
 
-   => '(1 2 3)
-   (1 2 3)
+Set variables with :ref:`setv`::
 
-You also have access to all the built-in types' nice methods::
+    (setv zone-plane 8)
 
-  => (.strip " fooooo   ")
-  "fooooo"
+Access the elements of a list, dictionary, or other data structure with
+:ref:`get`::
 
-What's this?  Yes indeed, this is precisely the same as::
+    (setv fruit ["apple" "banana" "cantaloupe"])
+    (print (get fruit 0))  ; => apple
+    (setv (get fruit 1) "durian")
+    (print (get fruit 1))  ; => durian
 
-  " fooooo   ".strip()
+Access a range of elements in an ordered structure with :ref:`cut`::
 
-That's right---Lisp with dot notation!  If we have this string
-assigned as a variable, we can also do the following:
+    (print (cut "abcdef" 1 4))  ; => bcd
 
-.. code-block:: clj
+Conditional logic can be built with :ref:`if`::
 
-   (setv this-string " fooooo   ")
-   (this-string.strip)
+    (if (= 1 1)
+      (print "Math works. The universe is safe.")
+      (print "Math has failed. The universe is doomed."))
 
-What about conditionals?:
+As in this example, ``if`` is called like ``(if CONDITION THEN ELSE)``. It
+executes and returns the form ``THEN`` if ``CONDITION`` is true (according to
+:class:`bool`) and ``ELSE`` otherwise. If ``ELSE`` is omitted, ``None`` is used
+in its place.
 
-.. code-block:: clj
+What if you want to use more than form in place of the ``THEN`` or ``ELSE``
+clauses, or in place of ``CONDITION``, for that matter? Use the special
+operator :ref:`do` (known more traditionally in Lisp as ``progn``), which
+combines several forms into one, returning the last::
 
-   (if (try-some-thing)
-     (print "this is if true")
-     (print "this is if false"))
+   (if (do (print "Let's check.") (= 1 1))
+     (do
+       (print "Math works.")
+       (print "The universe is safe."))
+     (do
+       (print "Math has failed.")
+       (print "The universe is doomed.")))
 
-As you can tell above, the first argument to ``if`` is a truth test, the
-second argument is the body if true, and the third argument (optional!)
-is if false (ie. ``else``).
-
-If you need to do more complex conditionals, you'll find that you
-don't have ``elif`` available in Hy.  Instead, you should use something
-called ``cond``.  In Python, you might do something like::
-
-  somevar = 33
-  if somevar > 50:
-      print("That variable is too big!")
-  elif somevar < 10:
-      print("That variable is too small!")
-  else:
-      print("That variable is jussssst right!")
-
-In Hy, you would do:
-
-.. code-block:: clj
+For branching on more than one case, try :ref:`cond`::
 
    (setv somevar 33)
    (cond
@@ -278,306 +180,140 @@ In Hy, you would do:
     [True
      (print "That variable is jussssst right!")])
 
-What you'll notice is that ``cond`` switches off between a statement
-that is executed and checked conditionally for true or falseness, and
-then a bit of code to execute if it turns out to be true.  You'll also
-notice that the ``else`` is implemented at the end simply by checking
-for ``True`` -- that's because ``True`` will always be true, so if we get
-this far, we'll always run that one!
+The macro ``(when CONDITION THEN-1 THEN-2 …)`` is shorthand for ``(if CONDITION
+(do THEN-1 THEN-2 …))``. ``unless`` works the same as ``when``, but inverts the
+condition with ``not``.
 
-You might notice above that if you have code like:
+Hy's basic loops are :ref:`while` and :ref:`for`::
 
-.. code-block:: clj
+    (setv x 3)
+    (while (> x 0)
+      (print x)
+      (setv x (- x 1)))  ; => 3 2 1
 
-   (if some-condition
-     (body-if-true)
-     (body-if-false))
+    (for [x [1 2 3]]
+      (print x))         ; => 1 2 3
 
-But wait!  What if you want to execute more than one statement in the
-body of one of these?
+A more functional way to iterate is provided by the comprehension forms such as
+:ref:`lfor`. Whereas ``for`` always returns ``None``, ``lfor`` returns a list
+with one element per iteration. ::
 
-You can do the following:
+    (print (lfor  x [1 2 3]  (* x 2)))  ; => [2, 4, 6]
 
-.. code-block:: clj
 
-   (if (try-some-thing)
-     (do
-       (print "this is if true")
-       (print "and why not, let's keep talking about how true it is!"))
-     (print "this one's still simply just false"))
+Functions, classes, and modules
+===============================
 
-You can see that we used ``do`` to wrap multiple statements.  If you're
-familiar with other Lisps, this is the equivalent of ``progn``
-elsewhere.
+Define named functions with :ref:`defn`::
 
-Comments start with semicolons:
+    (defn fib [n]
+      (if (< n 2)
+        n
+        (+ (fib (- n 1)) (fib (- n 2)))))
+    (print (fib 8))  ; => 21
 
-.. code-block:: clj
+Define anonymous functions with :ref:`fn`::
 
-  (print "this will run")
-  ; (print "but this will not")
-  (+ 1 2 3)  ; we'll execute the addition, but not this comment!
+    (print (list (filter (fn [x] (% x 2)) (range 10))))
+      ; => [1, 3, 5, 7, 9]
 
-Hashbang (``#!``) syntax is supported:
+Special symbols in the parameter list of ``defn`` or ``fn`` allow you to
+indicate optional arguments, provide default values, and collect unlisted
+arguments::
 
-.. code-block:: clj
+    (defn test [a b &optional c [d "x"] &rest e]
+      [a b c d e])
+    (print (test 1 2))            ; => [1, 2, None, 'x', ()]
+    (print (test 1 2 3 4 5 6 7))  ; => [1, 2, 3, 4, (5, 6, 7)]
 
-   #! /usr/bin/env hy
-   (print "Make me executable, and run me!")
+Set a function parameter by name with a ``:keyword``::
 
-Looping is not hard but has a kind of special structure.  In Python,
-we might do::
+    (test 1 2 :d "y")             ; => [1, 2, None, 'y', ()]
 
-  for i in range(10):
-      print("'i' is now at " + str(i))
+Define classes with :ref:`defclass`::
 
-The equivalent in Hy would be:
+    (defclass FooBar []
+      (defn __init__ [self x]
+        (setv self.x x))
+      (defn get-x [self]
+        self.x))
 
-.. code-block:: clj
+Here we create a new instance ``fb`` of ``FooBar`` and access its attributes by
+various means::
 
-  (for [i (range 10)]
-    (print (+ "'i' is now at " (str i))))
+    (setv fb (FooBar 15))
+    (print fb.x)         ; => 15
+    (print (. fb x))     ; => 15
+    (print (.get-x fb))  ; => 15
+    (print (fb.get-x))   ; => 15
 
-Python's collections indexes and slices are implemented
-by the ``get`` and ``cut`` built-in:
+Note that syntax like ``fb.x`` and ``fb.get-x`` only works when the object
+being invoked (``fb``, in this case) is a simple variable name. To get an
+attribute or call a method of an arbitrary form ``FORM``, you must use the
+syntax ``(. FORM x)`` or ``(.get-x FORM)``.
 
-.. code-block:: clj
+Access an external module, whether written in Python or Hy, with
+:ref:`import`::
 
-  (setv array [0 1 2])
-  (get array 1)
-  (cut array -3 -1)
+    (import math)
+    (print (math.sqrt 2))  ; => 1.4142135623730951
 
-which is equivalent to::
-
-  array[1]
-  array[-3:-1]
-
-You can also import and make use of various Python libraries.  For
-example:
-
-.. code-block:: clj
-
-   (import os)
-
-   (if (os.path.isdir "/tmp/somedir")
-     (os.mkdir "/tmp/somedir/anotherdir")
-     (print "Hey, that path isn't there!"))
-
-Python's context managers (``with`` statements) are used like this:
-
-.. code-block:: clj
-
-     (with [f (open "/tmp/data.in")]
-       (print (.read f)))
-
-which is equivalent to::
-
-  with open("/tmp/data.in") as f:
-      print(f.read())
-
-And yes, we do have List comprehensions!  In Python you might do::
-
-  odds_squared = [
-    pow(num, 2)
-    for num in range(100)
-    if num % 2 == 1]
-
-In Hy, you could do these like:
-
-.. code-block:: clj
-
-  (setv odds-squared
-    (lfor
-      num (range 100)
-      :if (= (% num 2) 1)
-      (pow num 2)))
-
-.. code-block:: clj
-
-  ; And, an example stolen shamelessly from a Clojure page:
-  ; Let's list all the blocks of a Chessboard:
-
-  (lfor
-    x (range 8)
-    y "ABCDEFGH"
-    (, x y))
-
-  ; [(0, 'A'), (0, 'B'), (0, 'C'), (0, 'D'), (0, 'E'), (0, 'F'), (0, 'G'), (0, 'H'),
-  ;  (1, 'A'), (1, 'B'), (1, 'C'), (1, 'D'), (1, 'E'), (1, 'F'), (1, 'G'), (1, 'H'),
-  ;  (2, 'A'), (2, 'B'), (2, 'C'), (2, 'D'), (2, 'E'), (2, 'F'), (2, 'G'), (2, 'H'),
-  ;  (3, 'A'), (3, 'B'), (3, 'C'), (3, 'D'), (3, 'E'), (3, 'F'), (3, 'G'), (3, 'H'),
-  ;  (4, 'A'), (4, 'B'), (4, 'C'), (4, 'D'), (4, 'E'), (4, 'F'), (4, 'G'), (4, 'H'),
-  ;  (5, 'A'), (5, 'B'), (5, 'C'), (5, 'D'), (5, 'E'), (5, 'F'), (5, 'G'), (5, 'H'),
-  ;  (6, 'A'), (6, 'B'), (6, 'C'), (6, 'D'), (6, 'E'), (6, 'F'), (6, 'G'), (6, 'H'),
-  ;  (7, 'A'), (7, 'B'), (7, 'C'), (7, 'D'), (7, 'E'), (7, 'F'), (7, 'G'), (7, 'H')]
-
-
-Python has support for various fancy argument and keyword arguments.
-In Python we might see::
-
-  >>> def optional_arg(pos1, pos2, keyword1=None, keyword2=42):
-  ...   return [pos1, pos2, keyword1, keyword2]
-  ...
-  >>> optional_arg(1, 2)
-  [1, 2, None, 42]
-  >>> optional_arg(1, 2, 3, 4)
-  [1, 2, 3, 4]
-  >>> optional_arg(keyword1=1, pos2=2, pos1=3, keyword2=4)
-  [3, 2, 1, 4]
-
-The same thing in Hy::
-
-  => (defn optional-arg [pos1 pos2 &optional keyword1 [keyword2 42]]
-  ...  [pos1 pos2 keyword1 keyword2])
-  => (optional-arg 1 2)
-  [1 2 None 42]
-  => (optional-arg 1 2 3 4)
-  [1 2 3 4]
-
-You can call keyword arguments like this::
-
-  => (optional-arg :keyword1 1
-  ...              :pos2 2
-  ...              :pos1 3
-  ...              :keyword2 4)
-  [3, 2, 1, 4]
-
-You can unpack arguments with the syntax ``#* args`` and ``#** kwargs``,
-similar to `*args` and `**kwargs` in Python::
-
-  => (setv args [1 2])
-  => (setv kwargs {"keyword2" 3
-  ...              "keyword1" 4})
-  => (optional-arg #* args #** kwargs)
-  [1, 2, 4, 3]
-
-Hy also supports ``*args`` and ``**kwargs`` in parameter lists.  In Python::
-
-  def some_func(foo, bar, *args, **kwargs):
-    import pprint
-    pprint.pprint((foo, bar, args, kwargs))
-
-The Hy equivalent:
-
-.. code-block:: clj
-
-  (defn some-func [foo bar &rest args &kwargs kwargs]
-    (import pprint)
-    (pprint.pprint (, foo bar args kwargs)))
-
-Finally, of course we need classes!  In Python, we might have a class
-like::
-
-  class FooBar(object):
-      """
-      Yet Another Example Class
-      """
-      def __init__(self, x):
-          self.x = x
-
-      def get_x(self):
-          """
-          Return our copy of x
-          """
-          return self.x
-
-And we might use it like::
-
-  bar = FooBar(1)
-  print(bar.get_x())
-
-
-In Hy:
-
-.. code-block:: clj
-
-  (defclass FooBar [object]
-    "Yet Another Example Class"
-
-    (defn --init-- [self x]
-      (setv self.x x))
-
-    (defn get-x [self]
-      "Return our copy of x"
-      self.x))
-
-And we can use it like:
-
-.. code-block:: clj
-
-  (setv bar (FooBar 1))
-  (print (bar.get-x))
-
-Or using the leading dot syntax!
-
-.. code-block:: clj
-
-  (print (.get-x (FooBar 1)))
-
-
-You can also do class-level attributes.  In Python::
-
-  class Customer(models.Model):
-      name = models.CharField(max_length=255)
-      address = models.TextField()
-      notes = models.TextField()
-
-In Hy:
-
-.. code-block:: clj
-
-  (defclass Customer [models.Model]
-     (setv name (models.CharField :max-length 255))
-     (setv address (models.TextField))
-     (setv notes (models.TextField)))
+Python can import a Hy module like any other module so long as Hy itself has
+been imported first, which, of course, must have already happened if you're
+running a Hy program.
 
 Macros
 ======
 
-One really powerful feature of Hy are macros. They are small functions that are
-used to generate code (or data). When a program written in Hy is started, the
-macros are executed and their output is placed in the program source. After this,
-the program starts executing normally. Very simple example:
+Macros are the basic metaprogramming tool of Lisp. A macro is a function that
+is called at compile time (i.e., when a Hy program is being translated to
+Python :mod:`ast` objects) and returns code, which becomes part of the final
+program. Here's a simple example::
 
-.. code-block:: clj
+    (print "Executing")
+    (defmacro m []
+      (print "Now for a slow computation")
+      (setv x (% (** 10 10 7) 3))
+      (print "Done computing")
+      x)
+    (print "Value:" (m))
+    (print "Done executing")
 
-  => (defmacro hello [person]
-  ...  `(print "Hello there," ~person))
-  => (hello "Tuukka")
-  Hello there, Tuukka
+If you run this program twice in a row, you'll see this::
 
-The thing to notice here is that hello macro doesn't output anything on
-screen. Instead it creates piece of code that is then executed and prints on
-screen. This macro writes a piece of program that looks like this (provided that
-we used "Tuukka" as parameter):
+    $ hy example.hy
+    Now for a slow computation
+    Done computing
+    Executing
+    Value: 1
+    Done executing
+    $ hy example.hy 
+    Executing
+    Value: 1
+    Done executing
 
-.. code-block:: clj
+The slow computation is performed while compiling the program on its first
+invocation. Only after the whole program is compiled does normal execution
+begin from the top, printing "Executing". When the program is called a second
+time, it is run from the previously compiled bytecode, which is equivalent to
+simply::
 
-  (print "Hello there," "Tuukka")
+    (print "Executing")
+    (print "Value:" 1)
+    (print "Done executing")
 
-We can also manipulate code with macros:
-
-.. code-block:: clj
-
-  => (defmacro rev [code]
-  ...  (setv op (last code) params (list (butlast code)))
-  ...  `(~op ~@params))
-  => (rev (1 2 3 +))
-  6
-
-The code that was generated with this macro just switched around some of the
-elements, so by the time program started executing, it actually reads:
-
-.. code-block:: clj
-
-  (+ 1 2 3)
+Our macro ``m`` has an especially simple return value, an integer, which at
+compile-time is converted to an integer literal. In general, macros can return
+arbitrary Hy forms to be executed as code. There are several special operators
+and macros that make it easy to construct forms programmatically, such as
+:ref:`quote` (``'``), :ref:`quasiquote` (`````), :ref:`unquote` (``~``), and
+:ref:`defmacro!`. The previous chapter has :ref:`a simple example <do-while>`
+of using ````` and ``~`` to define a new control construct ``do-while``.
 
 Sometimes it's nice to be able to call a one-parameter macro without
-parentheses. Tag macros allow this. The name of a tag macro is typically
-one character long, but since Hy operates well with Unicode, we aren't running
-out of characters that soon:
-
-.. code-block:: clj
+parentheses. Tag macros allow this. The name of a tag macro is often just one
+character long, but since Hy allows most Unicode characters in the name of a
+macro (or ordinary variable), you won't out of characters soon. ::
 
   => (deftag ↻ [code]
   ...  (setv op (last code) params (list (butlast code)))
@@ -585,106 +321,23 @@ out of characters that soon:
   => #↻(1 2 3 +)
   6
 
-Macros are useful when one wishes to extend Hy or write their own
-language on top of that. Many features of Hy are macros, like ``when``,
-``cond`` and ``->``.
-
-What if you want to use a macro that's defined in a different
-module? The special form ``import`` won't help, because it merely
-translates to a Python ``import`` statement that's executed at
-run-time, and macros are expanded at compile-time, that is,
-during the translate from Hy to Python. Instead, use ``require``,
-which imports the module and makes macros available at
-compile-time. ``require`` uses the same syntax as ``import``.
-
-.. code-block:: clj
+What if you want to use a macro that's defined in a different module?
+``import`` won't help, because it merely translates to a Python ``import``
+statement that's executed at run-time, and macros are expanded at compile-time,
+that is, during the translation from Hy to Python. Instead, use :ref:`require`,
+which imports the module and makes macros available at compile-time.
+``require`` uses the same syntax as ``import``. ::
 
    => (require tutorial.macros)
    => (tutorial.macros.rev (1 2 3 +))
    6
 
-Hy <-> Python interop
-=====================
+Next steps
+==========
 
-Using Hy from Python
---------------------
+You now know enough to be dangerous with Hy. You may now smile villainously and
+sneak off to your Hydeaway to do unspeakable things.
 
-You can use Hy modules in Python!
-
-If you save the following in ``greetings.hy``:
-
-.. code-block:: clj
-
-    (defn greet [name] (print "hello from hy," name))
-
-Then you can use it directly from Python, by importing Hy before importing
-the module. In Python::
-
-    import hy
-    import greetings
-
-    greetings.greet("Foo")
-
-Using Python from Hy
---------------------
-
-You can also use any Python module in Hy!
-
-If you save the following in ``greetings.py`` in Python::
-
-    def greet(name):
-        print("hello, %s" % (name))
-
-You can use it in Hy (see :ref:`import`):
-
-.. code-block:: clj
-
-    (import greetings)
-    (.greet greetings "foo")
-
-More information on :doc:`../language/interop`.
-
-
-Protips!
-========
-
-Hy also features something known as the "threading macro", a really neat
-feature of Clojure's. The "threading macro" (written as ``->``) is used
-to avoid deep nesting of expressions.
-
-The threading macro inserts each expression into the next expression's first
-argument place.
-
-Let's take the classic:
-
-.. code-block:: clj
-
-    (require [hy.contrib.loop [loop]])
-
-    (loop (print (eval (read))))
-
-Rather than write it like that, we can write it as follows:
-
-.. code-block:: clj
-
-    (require [hy.contrib.loop [loop]])
-
-    (-> (read) (eval) (print) (loop))
-
-Now, using `python-sh <http://amoffat.github.com/sh/>`_, we can show
-how the threading macro (because of python-sh's setup) can be used like
-a pipe:
-
-.. code-block:: clj
-
-    => (import [sh [cat grep wc]])
-    => (-> (cat "/usr/share/dict/words") (grep "-E" "^hy") (wc "-l"))
-    210
-
-Which, of course, expands out to:
-
-.. code-block:: clj
-
-    (wc (grep (cat "/usr/share/dict/words") "-E" "^hy") "-l")
-
-Much more readable, no? Use the threading macro!
+Refer to Python's documention for the details of Python semantics, and the rest
+of this manual for Hy-specific features. Like Hy itself, the manual is
+incomplete, but :ref:`contributions <hacking>` are always welcome.
diff --git a/docs/whyhy.rst b/docs/whyhy.rst
new file mode 100644
index 0000000..55ad8bc
--- /dev/null
+++ b/docs/whyhy.rst
@@ -0,0 +1,142 @@
+=======
+Why Hy?
+=======
+
+Hy is a multi-paradigm general-purpose programming language in the `Lisp family
+<https://en.wikipedia.org/wiki/Lisp_(programming_language)>`_. It's implemented
+as a kind of alternative syntax for Python. Compared to Python, Hy offers a
+variety of extra features, generalizations, and syntactic simplifications, as
+would be expected of a Lisp. Compared to other Lisps, Hy provides direct access
+to Python's built-ins and third-party Python libraries, while allowing you to
+freely mix imperative, functional, and object-oriented styles of programming.
+
+
+Hy versus Python
+----------------
+
+The first thing a Python programmer will notice about Hy is that it has Lisp's
+traditional parenthesis-heavy prefix syntax in place of Python's C-like infix
+syntax. For example, ``print("The answer is", 2 + object.method(arg))`` could
+be written ``(print "The answer is" (+ 2 (.method object arg)))`` in Hy.
+Consequently, Hy is free-form: structure is indicated by parentheses rather
+than whitespace, making it convenient for command-line use.
+
+As in other Lisps, the value of a simplistic syntax is that it facilitates
+Lisp's signature feature: `metaprogramming
+<https://en.wikipedia.org/wiki/Metaprogramming>`_ through macros, which are
+functions that manipulate code objects at compile time to produce new code
+objects, which are then executed as if they had been part of the original code.
+In fact, Hy allows arbitrary computation at compile-time. For example, here's a
+simple macro that implements a C-style do-while loop, which executes its body
+for as long as the condition is true, but at least once.
+
+.. _do-while:
+
+::
+
+    (defmacro do-while [condition &rest body]
+      `(do
+        ~body
+        (while ~condition
+          ~body)))
+
+    (setv x 0)
+    (do-while x
+      (print "This line is executed once."))
+
+Hy also removes Python's restrictions on mixing expressions and statements,
+allowing for more direct and functional code. For example, Python doesn't allow
+:ref:`with <py:with>` blocks, which close a resource once you're done using it,
+to return values. They can only execute a set of statements:
+
+.. code-block:: python
+
+    with open("foo") as o:
+       f1 = o.read()
+    with open("bar") as o:
+       f2 = o.read()
+    print(len(f1) + len(f2))
+
+In Hy, :ref:`with` returns the value of its last body form, so you can use it
+like an ordinary function call::
+
+   (print (+
+     (len (with [o (open "foo")] (.read o))
+     (len (with [o (open "bar")] (.read o))))))
+
+To be even more concise, you can put a ``with`` form in a :ref:`generator
+expression <gfor>`::
+
+   (print (sum (gfor
+     filename ["foo" "bar"]
+     (len (with [o (open filename)] (.read o))))))
+
+Finally, Hy offers several generalizations to Python's binary operators.
+Operators can be given more than two arguments (e.g., ``(+ 1 2 3)``), including
+augmented assignment operators (e.g., ``(+= x 1 2 3)``). They are also provided
+as ordinary first-class functions of the same name, allowing them to be passed
+to higher-order functions: ``(sum xs)`` could be written ``(reduce + xs)``.
+
+The Hy compiler works by reading Hy source code into Hy model objects and
+compiling the Hy model objects into Python abstract syntax tree (:py:mod:`ast`)
+objects. Python AST objects can then be compiled and run by Python itself,
+byte-compiled for faster execution later, or rendered into Python source code.
+You can even :ref:`mix Python and Hy code in the same project <interop>`, which
+can be a good way to get your feet wet in Hy.
+
+
+Hy versus other Lisps
+---------------------
+
+At run-time, Hy is essentially Python code. Thus, while Hy's design owes a lot
+to `Clojure <https://clojure.org>`_, it is more tightly coupled to Python than
+Clojure is to Java; a better analogy is `CoffeeScript's
+<https://coffeescript.org>`_ relationship to JavaScript. Python's built-in
+:ref:`functions <py:built-in-funcs>` and :ref:`data structures
+<py:bltin-types>` are directly available::
+
+    (print (int "deadbeef" :base 16))  ; 3735928559
+    (print (len [1 10 100]))           ; 3
+
+The same goes for third-party Python libraries from `PyPI <https://pypi.org>`_
+and elsewhere. Here's a tiny `CherryPy <https://cherrypy.org>`_ web application
+in Hy::
+
+    (import cherrypy)
+
+    (defclass HelloWorld []
+      (#@ cherrypy.expose (defn index [self]
+        "Hello World!")))
+
+    (cherrypy.quickstart (HelloWorld))
+
+You can even run Hy on `PyPy <https://pypy.org>`_ for a particularly speedy
+Lisp.
+
+Like all Lisps, Hy is `homoiconic
+<https://en.wikipedia.org/wiki/Homoiconicity>`_. Its syntax is represented not
+with cons cells or with Python's basic data structures, but with simple
+subclasses of Python's basic data structures called :ref:`models <models>`.
+Using models in place of plain ``list``\s, ``set``\s, and so on has two
+purposes: models can keep track of their line and column numbers for the
+benefit of error messages, and models can represent syntactic features that the
+corresponding primitive type can't, such as the order in which elements appear
+in a set literal. However, models can be concatenated and indexed just like
+plain lists, and you can return ordinary Python types from a macro or give them
+to ``eval`` and Hy will automatically promote them to models.
+
+Hy takes much of its semantics from Python. For example, Hy is a Lisp-1 because
+Python functions use the same namespace as objects that aren't functions. In
+general, any Python code should be possible to literally translate to Hy. At
+the same time, Hy goes to some lengths to allow you to do typical Lisp things
+that aren't straightforward in Python. For example, Hy provides the
+aforementioned mixing of statements and expressions, :ref:`name mangling
+<mangling>` that transparently converts symbols with names like ``valid?`` to
+Python-legal identifiers, and a :ref:`let` macro to provide block-level scoping
+in place of Python's usual function-level scoping.
+
+Overall, Hy, like Common Lisp, is intended to be an unopinionated big-tent
+language that lets you do what you want. If you're interested in a more
+small-and-beautiful approach to Lisp, in the style of Scheme, check out
+`Hissp <https://github.com/gilch/hissp>`_, another Lisp embedded in Python
+that was created by a Hy developer.

From f227f689d95ee46525a034596071111bd255dc2a Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Wed, 11 Sep 2019 10:24:03 -0400
Subject: [PATCH 203/223] Advertise our Stack Overflow tag

---
 README.md      | 1 +
 docs/index.rst | 1 +
 2 files changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 68180d9..34b00f0 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,7 @@ Project
   * [Contributor Guidelines](http://docs.hylang.org/en/master/hacking.html#contributor-guidelines)
   * [Code of Conduct](http://docs.hylang.org/en/master/hacking.html#contributor-code-of-conduct)
 * IRC: Join #hy on [freenode](https://webchat.freenode.net/)
+* [Stack Overflow: The [hy] tag](https://stackoverflow.com/questions/tagged/hy)
 
 ![Cuddles the Hacker](https://i.imgur.com/QbPMXTN.png)
 
diff --git a/docs/index.rst b/docs/index.rst
index 6089770..c6991f9 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -9,6 +9,7 @@ The Hy Manual
 :Source: https://github.com/hylang/hy
 :List: `hylang-discuss <https://groups.google.com/forum/#!forum/hylang-discuss>`_
 :IRC: irc://chat.freenode.net/hy
+:Stack Overflow: `The [hy] tag <https://stackoverflow.com/questions/tagged/hy>`_
 
 Hy is a Lisp dialect that's embedded in Python. Since Hy transforms its Lisp
 code into Python abstract syntax tree (AST) objects, you have the whole

From 80771ac99c3886a89bda4c11e5a87a342004b15d Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 2 Sep 2019 14:36:44 -0400
Subject: [PATCH 204/223] Remove documentation of `print` in api.rst

There is no longer any such special form. We just use Python 3's built-in function.
---
 docs/language/api.rst | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index c4f877b..5fe8920 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -1406,18 +1406,6 @@ parameter will be returned.
     True
 
 
-print
------
-
-``print`` is used to output on screen. Example usage:
-
-.. code-block:: clj
-
-    (print "Hello world!")
-
-.. note:: ``print`` always returns ``None``.
-
-
 .. _quasiquote:
 
 quasiquote

From 8351ccf9d9a82409c202bc0f73eb28e3861f5415 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Fri, 6 Sep 2019 15:15:22 -0400
Subject: [PATCH 205/223] Allow inline Python

---
 NEWS.rst                    |  2 ++
 docs/language/api.rst       | 38 +++++++++++++++++++++++++++++++++++++
 docs/language/interop.rst   |  6 ++++--
 docs/whyhy.rst              |  4 ++--
 hy/compiler.py              | 17 +++++++++++++++++
 tests/compilers/test_ast.py |  7 +++++++
 tests/resources/pydemo.hy   | 11 +++++++++++
 tests/test_hy2py.py         |  8 +++++++-
 8 files changed, 88 insertions(+), 5 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index f11c5c8..3c7db04 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -13,6 +13,8 @@ Removals
 
 New Features
 ------------------------------
+* Added special forms ``py`` to ``pys`` that allow Hy programs to include
+  inline Python code.
 * All augmented assignment operators (except `%=` and `^=`) now allow
   more than two arguments.
 
diff --git a/docs/language/api.rst b/docs/language/api.rst
index 5fe8920..496f3da 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -1406,6 +1406,44 @@ parameter will be returned.
     True
 
 
+.. _py-specialform:
+
+py
+--
+
+``py`` parses the given Python code at compile-time and inserts the result into
+the generated abstract syntax tree. Thus, you can mix Python code into a Hy
+program. Only a Python expression is allowed, not statements; use
+:ref:`pys-specialform` if you want to use Python statements. The value of the
+expression is returned from the ``py`` form. ::
+
+    (print "A result from Python:" (py "'hello' + 'world'"))
+
+The code must be given as a single string literal, but you can still use
+macros, :ref:`eval`, and related tools to construct the ``py`` form. If you
+want to evaluate some Python code that's only defined at run-time, try the
+standard Python function :func:`eval`.
+
+Python code need not syntactically round-trip if you use ``hy2py`` on a Hy
+program that uses ``py`` or ``pys``. For example, comments will be removed.
+
+
+.. _pys-specialform:
+
+pys
+---
+
+As :ref:`py-specialform`, but the code can consist of zero or more statements,
+including compound statements such as ``for`` and ``def``. ``pys`` always
+returns ``None``. Also, the code string is dedented with
+:func:`textwrap.dedent` before parsing, which allows you to intend the code to
+match the surrounding Hy code, but significant leading whitespace in embedded
+string literals will be removed. ::
+
+    (pys "myvar = 5")
+    (print "myvar is" myvar)
+
+
 .. _quasiquote:
 
 quasiquote
diff --git a/docs/language/interop.rst b/docs/language/interop.rst
index cb009b4..d4079d3 100644
--- a/docs/language/interop.rst
+++ b/docs/language/interop.rst
@@ -19,9 +19,11 @@ Hy and Python. For example, Python's ``str.format_map`` can be written
 Using Python from Hy
 ====================
 
-Using Python from Hy is nice and easy, you just have to :ref:`import` it.
+You can embed Python code directly into a Hy program with the special operators
+:ref:`py-specialform` and :ref:`pys-specialform`.
 
-If you have the following in ``greetings.py`` in Python::
+Using a Python module from Hy is nice and easy: you just have to :ref:`import`
+it. If you have the following in ``greetings.py`` in Python::
 
     def greet(name):
         print("hello," name)
diff --git a/docs/whyhy.rst b/docs/whyhy.rst
index 55ad8bc..cde6ee3 100644
--- a/docs/whyhy.rst
+++ b/docs/whyhy.rst
@@ -81,8 +81,8 @@ The Hy compiler works by reading Hy source code into Hy model objects and
 compiling the Hy model objects into Python abstract syntax tree (:py:mod:`ast`)
 objects. Python AST objects can then be compiled and run by Python itself,
 byte-compiled for faster execution later, or rendered into Python source code.
-You can even :ref:`mix Python and Hy code in the same project <interop>`, which
-can be a good way to get your feet wet in Hy.
+You can even :ref:`mix Python and Hy code in the same project, or even the same
+file,<interop>` which can be a good way to get your feet wet in Hy.
 
 
 Hy versus other Lisps
diff --git a/hy/compiler.py b/hy/compiler.py
index 7b48822..26ff7e1 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -20,6 +20,7 @@ from hy.macros import require, load_macros, macroexpand, tag_macroexpand
 import hy.core
 
 import re
+import textwrap
 import pkgutil
 import traceback
 import importlib
@@ -1589,6 +1590,22 @@ class HyASTCompiler(object):
                 if ast_str(root) == "eval_and_compile"
                 else Result())
 
+    @special(["py", "pys"], [STR])
+    def compile_inline_python(self, expr, root, code):
+        exec_mode = root == HySymbol("pys")
+
+        try:
+            o = ast.parse(
+                textwrap.dedent(code) if exec_mode else code,
+                self.filename,
+                'exec' if exec_mode else 'eval').body
+        except (SyntaxError, ValueError if PY36 else TypeError) as e:
+            raise self._syntax_error(
+                expr,
+                "Python parse error in '{}': {}".format(root, e))
+
+        return Result(stmts=o) if exec_mode else o
+
     @builds_model(HyExpression)
     def compile_expression(self, expr):
         # Perform macro expansions
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 70ac563..e5e1d2d 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -639,3 +639,10 @@ def test_futures_imports():
 
     assert hy_ast.body[0].module == '__future__'
     assert hy_ast.body[1].module == 'hy.core.language'
+
+
+def test_inline_python():
+    can_compile('(py "1 + 1")')
+    cant_compile('(py "1 +")')
+    can_compile('(pys "if 1:\n  2")')
+    cant_compile('(pys "if 1\n  2")')
diff --git a/tests/resources/pydemo.hy b/tests/resources/pydemo.hy
index a23e3c2..5774dac 100644
--- a/tests/resources/pydemo.hy
+++ b/tests/resources/pydemo.hy
@@ -156,3 +156,14 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little
 (with [c1 (closing (Closeable)) c2 (closing (Closeable))]
   (setv c1.x "v1")
   (setv c2.x "v2"))
+(setv closed1 (.copy closed))
+
+(pys "
+  closed = []
+  pys_accum = []
+  for i in range(5):
+      with closing(Closeable()) as o:
+          class C: pass
+          o.x = C()
+      pys_accum.append(i)")
+(setv py-accum (py "''.join(map(str, pys_accum))"))
diff --git a/tests/test_hy2py.py b/tests/test_hy2py.py
index d19e4f6..f774b7d 100644
--- a/tests/test_hy2py.py
+++ b/tests/test_hy2py.py
@@ -120,4 +120,10 @@ def assert_stuff(m):
     assert issubclass(m.C2, m.C1)
     assert (m.C2.attr1, m.C2.attr2) == (5, 6)
 
-    assert m.closed == ["v2", "v1"]
+    assert m.closed1 == ["v2", "v1"]
+
+    assert len(m.closed) == 5
+    for a, b in itertools.combinations(m.closed, 2):
+        assert type(a) is not type(b)
+    assert m.pys_accum == [0, 1, 2, 3, 4]
+    assert m.py_accum == "01234"

From 31041f1713f9a22176b3610a9b619db52004ddb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Tempel?= <soeren+git@soeren-tempel.net>
Date: Mon, 23 Sep 2019 20:35:48 +0200
Subject: [PATCH 206/223] get_version.py: allow specifying version in
 environment variable

While packaging hy for Alpine Linux I noticed that the VERSIONFILE is
ignored if hy is build in a git repository. On Alpine the packages are
build in the package repository resulting in a wrong hy version. This
change allows specifying the version in an environment variable which is
preferred over git-describe(1) and the VERSIONFILE.
---
 get_version.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/get_version.py b/get_version.py
index 759336d..9224f59 100644
--- a/get_version.py
+++ b/get_version.py
@@ -5,16 +5,19 @@ import os, subprocess, runpy
 os.chdir(os.path.split(os.path.abspath(__file__))[0])
 VERSIONFILE = os.path.join("hy", "version.py")
 
-try:
-    __version__ = (subprocess.check_output
-                   (["git", "describe", "--tags", "--dirty"])
-                   .decode('ASCII').strip()
-                   .replace('-', '+', 1).replace('-', '.'))
-    with open(VERSIONFILE, "wt") as o:
-        o.write("__version__ = {!r}\n".format(__version__))
+if "HY_VERSION" in os.environ:
+    __version__ = os.environ["HY_VERSION"]
+else:
+    try:
+        __version__ = (subprocess.check_output
+                       (["git", "describe", "--tags", "--dirty"])
+                       .decode('ASCII').strip()
+                       .replace('-', '+', 1).replace('-', '.'))
+        with open(VERSIONFILE, "wt") as o:
+            o.write("__version__ = {!r}\n".format(__version__))
 
-except (subprocess.CalledProcessError, OSError):
-    if os.path.exists(VERSIONFILE):
-        __version__ = runpy.run_path(VERSIONFILE)['__version__']
-    else:
-        __version__ = "unknown"
+    except (subprocess.CalledProcessError, OSError):
+        if os.path.exists(VERSIONFILE):
+            __version__ = runpy.run_path(VERSIONFILE)['__version__']
+        else:
+            __version__ = "unknown"

From f5977555f4db68bf20182156e78cf60a213698ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B6ren=20Tempel?= <soeren+git@soeren-tempel.net>
Date: Mon, 23 Sep 2019 21:54:46 +0200
Subject: [PATCH 207/223] =?UTF-8?q?Add=20S=C3=B6ren=20Tempel=20to=20AUTHOR?=
 =?UTF-8?q?S?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7e744e0..b987a9b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -92,3 +92,4 @@
 * Brandon T. Willard <brandonwillard@gmail.com>
 * Andrew R. M. <nixy@nixy.moe>
 * Tristan de Cacqueray <tdecacqu@redhat.com>
+* Sören Tempel <soeren@soeren-tempel.net>

From 06213cd46c237cf61a3a7734ca30798c0d093c0d Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Tue, 8 Oct 2019 09:37:42 -0500
Subject: [PATCH 208/223] Remove some trailing whitespace from docs

---
 docs/tutorial.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
index 566417a..230e848 100644
--- a/docs/tutorial.rst
+++ b/docs/tutorial.rst
@@ -287,7 +287,7 @@ If you run this program twice in a row, you'll see this::
     Executing
     Value: 1
     Done executing
-    $ hy example.hy 
+    $ hy example.hy
     Executing
     Value: 1
     Done executing

From 6eedb19a07fc1fbd71cd5a648b7481d024a8f3e5 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Mon, 23 Sep 2019 12:21:06 -0500
Subject: [PATCH 209/223] Clean up assignment handling code

---
 hy/compiler.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 26ff7e1..612d429 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1336,15 +1336,17 @@ class HyASTCompiler(object):
 
     @special("setv", [many(FORM + FORM)])
     @special((PY38, "setx"), [times(1, 1, SYM + FORM)])
-    def compile_def_expression(self, expr, root, pairs):
-        if not pairs:
+    def compile_def_expression(self, expr, root, decls):
+        if not decls:
             return asty.Name(expr, id='None', ctx=ast.Load())
+
         result = Result()
-        for pair in pairs:
-            result += self._compile_assign(root, *pair)
+        is_assignment_expr = root == HySymbol("setx")
+        for decl in decls:
+            result += self._compile_assign(*decl, is_assignment_expr=is_assignment_expr)
         return result
 
-    def _compile_assign(self, root, name, result):
+    def _compile_assign(self, name, result, *, is_assignment_expr=False):
 
         if name in [HySymbol(x) for x in ("None", "True", "False")]:
             raise self._syntax_error(name,
@@ -1361,13 +1363,13 @@ class HyASTCompiler(object):
                 and isinstance(name, HySymbol)
                 and '.' not in name):
             result.rename(name)
-            if root != HySymbol("setx"):
+            if not is_assignment_expr:
                 # Throw away .expr to ensure that (setv ...) returns None.
                 result.expr = None
         else:
             st_name = self._storeize(name, ld_name)
             node = (asty.NamedExpr
-                if root == HySymbol("setx")
+                if is_assignment_expr
                 else asty.Assign)
             result += node(
                 name if hasattr(name, "start_line") else result,

From fd8514718b337b7bd387a129c9d7d9ff4a1eb752 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Mon, 23 Sep 2019 12:28:39 -0500
Subject: [PATCH 210/223] Refactor function argument compilation

---
 hy/compiler.py | 73 +++++++++++++++++++++++++++++++-------------------
 1 file changed, 46 insertions(+), 27 deletions(-)

diff --git a/hy/compiler.py b/hy/compiler.py
index 612d429..d908282 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -1440,27 +1440,32 @@ class HyASTCompiler(object):
             maybe(sym("&kwargs") + NASYM)),
         many(FORM)])
     def compile_function_def(self, expr, root, params, body):
-
         force_functiondef = root in ("fn*", "fn/a")
         node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef
+        ret = Result()
 
         mandatory, optional, rest, kwonly, kwargs = params
-        optional, defaults, ret = self._parse_optional_args(optional)
-        kwonly, kw_defaults, ret2 = self._parse_optional_args(kwonly, True)
-        ret += ret2
-        main_args = mandatory + optional
 
-        main_args, kwonly, [rest], [kwargs] = (
-            [[x and asty.arg(x, arg=ast_str(x), annotation=None)
-              for x in o]
-             for o in (main_args or [], kwonly or [], [rest], [kwargs])])
+        optional = optional or []
+        kwonly = kwonly or []
+
+        mandatory_ast, _, ret = self._compile_arguments_set(mandatory, False, ret)
+        optional_ast, optional_defaults, ret = self._compile_arguments_set(optional, True, ret)
+        kwonly_ast, kwonly_defaults, ret = self._compile_arguments_set(kwonly, False, ret)
+
+        rest_ast = kwargs_ast = None
+
+        if rest is not None:
+            [rest_ast], _, ret = self._compile_arguments_set([rest], False, ret)
+        if kwargs is not None:
+            [kwargs_ast], _, ret = self._compile_arguments_set([kwargs], False, ret)
 
         args = ast.arguments(
-            args=main_args, defaults=defaults,
-            vararg=rest,
+            args=mandatory_ast + optional_ast, defaults=optional_defaults,
+            vararg=rest_ast,
             posonlyargs=[],
-            kwonlyargs=kwonly, kw_defaults=kw_defaults,
-            kwarg=kwargs)
+            kwonlyargs=kwonly_ast, kw_defaults=kwonly_defaults,
+            kwarg=kwargs_ast)
 
         body = self._compile_branch(body)
 
@@ -1482,21 +1487,35 @@ class HyASTCompiler(object):
         ret += Result(expr=ast_name, temp_variables=[ast_name, ret.stmts[-1]])
         return ret
 
-    def _parse_optional_args(self, expr, allow_no_default=False):
-        # [a b [c 5] d] → ([a, b, c, d], [None, None, 5, d], <ret>)
-        names, defaults, ret = [], [], Result()
-        for x in expr or []:
-            sym, value = (
-                x if isinstance(x, HyList)
-                else (x, None) if allow_no_default
-                else (x, HySymbol('None').replace(x)))
-            names.append(sym)
-            if value is None:
-                defaults.append(None)
+    def _compile_arguments_set(self, decls, implicit_default_none, ret):
+        args_ast = []
+        args_defaults = []
+
+        for decl in decls:
+            default = None
+
+            # funcparserlib will check to make sure that the only times we
+            # ever have a HyList here are due to a default value.
+            if isinstance(decl, HyList):
+                sym, default = decl
             else:
-                ret += self.compile(value)
-                defaults.append(ret.force_expr)
-        return names, defaults, ret
+                sym = decl
+                if implicit_default_none:
+                    default = HySymbol('None').replace(sym)
+
+            if default is not None:
+                ret += self.compile(default)
+                args_defaults.append(ret.force_expr)
+            else:
+                # Note that the only time any None should ever appear here
+                # is in kwargs, since the order of those with defaults vs
+                # those without isn't significant in the same way as
+                # positional args.
+                args_defaults.append(None)
+
+            args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=None))
+
+        return args_ast, args_defaults, ret
 
     @special("return", [maybe(FORM)])
     def compile_return(self, expr, root, arg):

From 1865feb7d661e8db0323301f274f7b56856f617b Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Mon, 12 Aug 2019 16:09:32 -0500
Subject: [PATCH 211/223] Implement PEP 3107 & 526 annotations (closes #1794)

---
 NEWS.rst                              |   1 +
 hy/compiler.py                        | 152 ++++++++++++++++++++------
 hy/core/bootstrap.hy                  |   8 +-
 hy/lex/lexer.py                       |   4 +-
 hy/lex/parser.py                      |   6 +
 tests/native_tests/language.hy        |  26 ++++-
 tests/native_tests/py36_only_tests.hy |  10 ++
 7 files changed, 164 insertions(+), 43 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 3c7db04..e5b1d69 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -17,6 +17,7 @@ New Features
   inline Python code.
 * All augmented assignment operators (except `%=` and `^=`) now allow
   more than two arguments.
+* PEP 3107 and PEP 526 function and variable annotations are now supported.
 
 Other Breaking Changes
 ------------------------------
diff --git a/hy/compiler.py b/hy/compiler.py
index d908282..2b8bb6e 100755
--- a/hy/compiler.py
+++ b/hy/compiler.py
@@ -23,6 +23,7 @@ import re
 import textwrap
 import pkgutil
 import traceback
+import itertools
 import importlib
 import inspect
 import types
@@ -337,6 +338,15 @@ def mklist(*items, **kwargs):
    return make_hy_model(HyList, items, kwargs.get('rest'))
 
 
+# Parse an annotation setting.
+OPTIONAL_ANNOTATION = maybe(pexpr(sym("annotate*") + FORM) >> (lambda x: x[0]))
+
+
+def is_annotate_expression(model):
+    return (isinstance(model, HyExpression) and model and isinstance(model[0], HySymbol)
+            and model[0] == HySymbol("annotate*"))
+
+
 class HyASTCompiler(object):
     """A Hy-to-Python AST compiler"""
 
@@ -1334,7 +1344,7 @@ class HyASTCompiler(object):
         return ret + asty.AugAssign(
             expr, target=target, value=ret.force_expr, op=op())
 
-    @special("setv", [many(FORM + FORM)])
+    @special("setv", [many(OPTIONAL_ANNOTATION + FORM + FORM)])
     @special((PY38, "setx"), [times(1, 1, SYM + FORM)])
     def compile_def_expression(self, expr, root, decls):
         if not decls:
@@ -1343,21 +1353,43 @@ class HyASTCompiler(object):
         result = Result()
         is_assignment_expr = root == HySymbol("setx")
         for decl in decls:
-            result += self._compile_assign(*decl, is_assignment_expr=is_assignment_expr)
+            if is_assignment_expr:
+                ann = None
+                name, value = decl
+            else:
+                ann, name, value = decl
+
+            result += self._compile_assign(ann, name, value,
+                                           is_assignment_expr=is_assignment_expr)
         return result
 
-    def _compile_assign(self, name, result, *, is_assignment_expr=False):
+    @special(["annotate*"], [FORM, FORM])
+    def compile_basic_annotation(self, expr, root, ann, target):
+        return self._compile_assign(ann, target, None)
 
-        if name in [HySymbol(x) for x in ("None", "True", "False")]:
-            raise self._syntax_error(name,
-                "Can't assign to `{}'".format(name))
+    def _compile_assign(self, ann, name, value, *, is_assignment_expr = False):
+        # Ensure that assignment expressions have a result and no annotation.
+        assert not is_assignment_expr or (value is not None and ann is None)
 
-        result = self.compile(result)
         ld_name = self.compile(name)
 
-        if isinstance(ld_name.expr, ast.Call):
-            raise self._syntax_error(name,
-                "Can't assign to a callable: `{}'".format(name))
+        annotate_only = value is None
+        if annotate_only:
+            result = Result()
+        else:
+            result = self.compile(value)
+
+        invalid_name = False
+        if ann is not None:
+            # An annotation / annotated assignment is more strict with the target expression.
+            invalid_name = not isinstance(ld_name.expr, (ast.Name, ast.Attribute, ast.Subscript))
+        else:
+            invalid_name = (str(name) in ("None", "True", "False")
+                            or isinstance(ld_name.expr, ast.Call))
+
+        if invalid_name:
+            raise self._syntax_error(name, "illegal target for {}".format(
+                                        "annotation" if annotate_only else "assignment"))
 
         if (result.temp_variables
                 and isinstance(name, HySymbol)
@@ -1368,12 +1400,27 @@ class HyASTCompiler(object):
                 result.expr = None
         else:
             st_name = self._storeize(name, ld_name)
-            node = (asty.NamedExpr
-                if is_assignment_expr
-                else asty.Assign)
+
+            if ann is not None:
+                ann_result = self.compile(ann)
+                result = ann_result + result
+
+            if is_assignment_expr:
+                node = asty.NamedExpr
+            elif ann is not None:
+                if not PY36:
+                    raise self._syntax_error(name, "Variable annotations are not supported on "
+                                                   "Python <=3.6")
+
+                node = lambda x, **kw: asty.AnnAssign(x, annotation=ann_result.force_expr,
+                                                      simple=int(isinstance(name, HySymbol)),
+                                                      **kw)
+            else:
+                node = asty.Assign
+
             result += node(
                 name if hasattr(name, "start_line") else result,
-                value=result.force_expr,
+                value=result.force_expr if not annotate_only else None,
                 target=st_name, targets=[st_name])
 
         return result
@@ -1432,18 +1479,34 @@ class HyASTCompiler(object):
         # The starred version is for internal use (particularly, in the
         # definition of `defn`). It ensures that a FunctionDef is
         # produced rather than a Lambda.
+        OPTIONAL_ANNOTATION,
         brackets(
-            many(NASYM),
-            maybe(sym("&optional") + many(NASYM | brackets(SYM, FORM))),
-            maybe(sym("&rest") + NASYM),
-            maybe(sym("&kwonly") + many(NASYM | brackets(SYM, FORM))),
-            maybe(sym("&kwargs") + NASYM)),
+            many(OPTIONAL_ANNOTATION + NASYM),
+            maybe(sym("&optional") + many(OPTIONAL_ANNOTATION
+                                            + (NASYM | brackets(SYM, FORM)))),
+            maybe(sym("&rest") + OPTIONAL_ANNOTATION + NASYM),
+            maybe(sym("&kwonly") + many(OPTIONAL_ANNOTATION
+                                        + (NASYM | brackets(SYM, FORM)))),
+            maybe(sym("&kwargs") + OPTIONAL_ANNOTATION + NASYM)),
         many(FORM)])
-    def compile_function_def(self, expr, root, params, body):
+    def compile_function_def(self, expr, root, returns, params, body):
         force_functiondef = root in ("fn*", "fn/a")
         node = asty.AsyncFunctionDef if root == "fn/a" else asty.FunctionDef
         ret = Result()
 
+        # NOTE: Our evaluation order of return type annotations is
+        # different from Python: Python evalautes them after the argument
+        # annotations / defaults (as that's where they are in the source),
+        # but Hy evaluates them *first*, since here they come before the #
+        # argument list. Therefore, it would be more confusing for
+        # readability to evaluate them after like Python.
+
+        ret = Result()
+        returns_ann = None
+        if returns is not None:
+            returns_result = self.compile(returns)
+            ret += returns_result
+
         mandatory, optional, rest, kwonly, kwargs = params
 
         optional = optional or []
@@ -1469,7 +1532,7 @@ class HyASTCompiler(object):
 
         body = self._compile_branch(body)
 
-        if not force_functiondef and not body.stmts:
+        if not force_functiondef and not body.stmts and returns is None:
             return ret + asty.Lambda(expr, args=args, body=body.force_expr)
 
         if body.expr:
@@ -1481,7 +1544,8 @@ class HyASTCompiler(object):
                     name=name,
                     args=args,
                     body=body.stmts or [asty.Pass(expr)],
-                    decorator_list=[])
+                    decorator_list=[],
+                    returns=returns_result.force_expr if returns is not None else None)
 
         ast_name = asty.Name(expr, id=name, ctx=ast.Load())
         ret += Result(expr=ast_name, temp_variables=[ast_name, ret.stmts[-1]])
@@ -1491,7 +1555,7 @@ class HyASTCompiler(object):
         args_ast = []
         args_defaults = []
 
-        for decl in decls:
+        for ann, decl in decls:
             default = None
 
             # funcparserlib will check to make sure that the only times we
@@ -1503,6 +1567,12 @@ class HyASTCompiler(object):
                 if implicit_default_none:
                     default = HySymbol('None').replace(sym)
 
+            if ann is not None:
+                ret += self.compile(ann)
+                ann_ast = ret.force_expr
+            else:
+                ann_ast = None
+
             if default is not None:
                 ret += self.compile(default)
                 args_defaults.append(ret.force_expr)
@@ -1513,7 +1583,7 @@ class HyASTCompiler(object):
                 # positional args.
                 args_defaults.append(None)
 
-            args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=None))
+            args_ast.append(asty.arg(sym, arg=ast_str(sym), annotation=ann_ast))
 
         return args_ast, args_defaults, ret
 
@@ -1564,12 +1634,22 @@ class HyASTCompiler(object):
             return expr
 
         new_args = []
-        pairs = list(expr[1:])
-        while pairs:
-            k, v = (pairs.pop(0), pairs.pop(0))
+        decls = list(expr[1:])
+        while decls:
+            if is_annotate_expression(decls[0]):
+                # Handle annotations.
+                ann = decls.pop(0)
+            else:
+                ann = None
+
+            k, v = (decls.pop(0), decls.pop(0))
             if ast_str(k) == "__init__" and isinstance(v, HyExpression):
                 v += HyExpression([HySymbol("None")])
-            new_args.extend([k, v])
+
+            if ann is not None:
+                new_args.append(ann)
+
+            new_args.extend((k, v))
         return HyExpression([HySymbol("setv")] + new_args).replace(expr)
 
     @special("dispatch-tag-macro", [STR, FORM])
@@ -1628,7 +1708,7 @@ class HyASTCompiler(object):
         return Result(stmts=o) if exec_mode else o
 
     @builds_model(HyExpression)
-    def compile_expression(self, expr):
+    def compile_expression(self, expr, *, allow_annotation_expression=False):
         # Perform macro expansions
         expr = macroexpand(expr, self.module, self)
         if not isinstance(expr, HyExpression):
@@ -1644,17 +1724,20 @@ class HyASTCompiler(object):
         func = None
 
         if isinstance(root, HySymbol):
-
             # First check if `root` is a special operator, unless it has an
             # `unpack-iterable` in it, since Python's operators (`+`,
             # etc.) can't unpack. An exception to this exception is that
             # tuple literals (`,`) can unpack. Finally, we allow unpacking in
             # `.` forms here so the user gets a better error message.
             sroot = ast_str(root)
-            if (sroot in _special_form_compilers or sroot in _bad_roots) and (
+
+            bad_root = sroot in _bad_roots or (sroot == ast_str("annotate*")
+                                               and not allow_annotation_expression)
+
+            if (sroot in _special_form_compilers or bad_root) and (
                     sroot in (mangle(","), mangle(".")) or
                     not any(is_unpack("iterable", x) for x in args)):
-                if sroot in _bad_roots:
+                if bad_root:
                     raise self._syntax_error(expr,
                         "The special form '{}' is not allowed here".format(root))
                 # `sroot` is a special operator. Get the build method and
@@ -1705,6 +1788,11 @@ class HyASTCompiler(object):
                                        attr=ast_str(root),
                                        ctx=ast.Load())
 
+        elif is_annotate_expression(root):
+            # Flatten and compile the annotation expression.
+            ann_expr = HyExpression(root + args).replace(root)
+            return self.compile_expression(ann_expr, allow_annotation_expression=True)
+
         if not func:
             func = self.compile(root)
 
diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy
index d5b7e19..b8da6c7 100644
--- a/hy/core/bootstrap.hy
+++ b/hy/core/bootstrap.hy
@@ -60,14 +60,12 @@
 (defmacro macro-error [expression reason &optional [filename '--name--]]
   `(raise (hy.errors.HyMacroExpansionError ~reason ~filename ~expression None)))
 
-(defmacro defn [name lambda-list &rest body]
-  "Define `name` as a function with `lambda-list` signature and body `body`."
+(defmacro defn [name &rest args]
+  "Define `name` as a function with `args` as the signature, annotations, and body."
   (import hy)
   (if (not (= (type name) hy.HySymbol))
     (macro-error name "defn takes a name as first argument"))
-  (if (not (isinstance lambda-list hy.HyList))
-    (macro-error name "defn takes a parameter list as second argument"))
-  `(setv ~name (fn* ~lambda-list ~@body)))
+  `(setv ~name (fn* ~@args)))
 
 (defmacro defn/a [name lambda-list &rest body]
   "Define `name` as a function with `lambda-list` signature and body `body`."
diff --git a/hy/lex/lexer.py b/hy/lex/lexer.py
index f202d94..fddfe4b 100755
--- a/hy/lex/lexer.py
+++ b/hy/lex/lexer.py
@@ -10,7 +10,8 @@ lg = LexerGenerator()
 
 # A regexp for something that should end a quoting/unquoting operator
 # i.e. a space or a closing brace/paren/curly
-end_quote = r'(?![\s\)\]\}])'
+end_quote_set = r'\s\)\]\}'
+end_quote = r'(?![%s])' % end_quote_set
 
 identifier = r'[^()\[\]{}\'"\s;]+'
 
@@ -25,6 +26,7 @@ lg.add('QUOTE', r'\'%s' % end_quote)
 lg.add('QUASIQUOTE', r'`%s' % end_quote)
 lg.add('UNQUOTESPLICE', r'~@%s' % end_quote)
 lg.add('UNQUOTE', r'~%s' % end_quote)
+lg.add('ANNOTATION', r'\^(?![=%s])' % end_quote_set)
 lg.add('DISCARD', r'#_')
 lg.add('HASHSTARS', r'#\*+')
 lg.add('BRACKETSTRING', r'''(?x)
diff --git a/hy/lex/parser.py b/hy/lex/parser.py
index 1d5ce24..f3edb0e 100755
--- a/hy/lex/parser.py
+++ b/hy/lex/parser.py
@@ -134,6 +134,12 @@ def term_unquote_splice(state, p):
     return HyExpression([HySymbol("unquote-splice"), p[1]])
 
 
+@pg.production("term : ANNOTATION term")
+@set_quote_boundaries
+def term_annotation(state, p):
+    return HyExpression([HySymbol("annotate*"), p[1]])
+
+
 @pg.production("term : HASHSTARS term")
 @set_quote_boundaries
 def term_hashstars(state, p):
diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy
index 688f985..37dce9e 100644
--- a/tests/native_tests/language.hy
+++ b/tests/native_tests/language.hy
@@ -68,15 +68,15 @@
   "NATIVE: test that setv doesn't work on names Python can't assign to
   and that we can't mangle"
   (try (eval '(setv None 1))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(defn None [] (print "hello")))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(setv False 1))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(setv True 0))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e)))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e)))))
   (try (eval '(defn True [] (print "hello")))
-       (except [e [SyntaxError]] (assert (in "Can't assign to" (str e))))))
+       (except [e [SyntaxError]] (assert (in "illegal target for assignment" (str e))))))
 
 
 (defn test-setv-pairs []
@@ -908,6 +908,22 @@
   (assert (= mooey.__name__ "mooey")))
 
 
+(defn test-defn-annotations []
+  "NATIVE: test that annotations in defn work"
+
+  (defn f [^int p1 p2 ^str p3 &optional ^str o1 ^int [o2 0]
+           &rest ^str rest &kwonly ^str k1 ^int [k2 0] &kwargs ^bool kwargs])
+
+  (assert (= (. f __annotations__ ["p1"]) int))
+  (assert (= (. f __annotations__ ["p3"]) str))
+  (assert (= (. f __annotations__ ["o1"]) str))
+  (assert (= (. f __annotations__ ["o2"]) int))
+  (assert (= (. f __annotations__ ["rest"]) str))
+  (assert (= (. f __annotations__ ["k1"]) str))
+  (assert (= (. f __annotations__ ["k2"]) int))
+  (assert (= (. f __annotations__ ["kwargs"]) bool)))
+
+
 (defn test-return []
 
   ; `return` in main line
diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy
index 3ba0c4f..99f2144 100644
--- a/tests/native_tests/py36_only_tests.hy
+++ b/tests/native_tests/py36_only_tests.hy
@@ -6,6 +6,7 @@
 ;; conftest.py skips this file when running on Python <3.6.
 
 (import [asyncio [get-event-loop sleep]])
+(import [typing [get-type-hints]])
 
 
 (defn run-coroutine [coro]
@@ -38,6 +39,15 @@
         (else (setv x (+ x 50))))
       (assert (= x 53)))))
 
+(defn test-variable-annotations []
+  (defclass AnnotationContainer []
+    (setv ^int x 1 y 2)
+    (^bool z))
+
+  (setv annotations (get-type-hints AnnotationContainer))
+  (assert (= (get annotations "x") int))
+  (assert (= (get annotations "z") bool)))
+
 (defn test-pep-487 []
   (defclass QuestBase []
     (defn --init-subclass-- [cls swallow &kwargs kwargs]

From e1ab140a6e573bbfdf49412db12f94241557d755 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Mon, 7 Oct 2019 13:14:49 -0500
Subject: [PATCH 212/223] Implement the of macro

---
 hy/core/macros.hy                     | 14 ++++++++++++++
 tests/native_tests/py36_only_tests.hy |  7 ++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/hy/core/macros.hy b/hy/core/macros.hy
index e1d5d48..e8cda1d 100644
--- a/hy/core/macros.hy
+++ b/hy/core/macros.hy
@@ -139,6 +139,20 @@ the second form, the second result is inserted into the third form, and so on."
   ret)
 
 
+(defmacro of [base &rest args]
+  "Shorthand for indexing for type annotations.
+
+If only one arguments are given, this expands to just that argument. If two arguments are
+given, it expands to indexing the first argument via the second. Otherwise, the first argument
+is indexed using a tuple of the rest.
+
+E.g. `(of List int)` -> `List[int]`, `(of Dict str str)` -> `Dict[str, str]`."
+  (if
+    (empty? args) base
+    (= (len args) 1) `(get ~base ~@args)
+    `(get ~base (, ~@args))))
+
+
 (defmacro if-not [test not-branch &optional yes-branch]
   "Like `if`, but execute the first branch when the test fails"
   `(if* (not ~test) ~not-branch ~yes-branch))
diff --git a/tests/native_tests/py36_only_tests.hy b/tests/native_tests/py36_only_tests.hy
index 99f2144..9b112e5 100644
--- a/tests/native_tests/py36_only_tests.hy
+++ b/tests/native_tests/py36_only_tests.hy
@@ -6,7 +6,7 @@
 ;; conftest.py skips this file when running on Python <3.6.
 
 (import [asyncio [get-event-loop sleep]])
-(import [typing [get-type-hints]])
+(import [typing [get-type-hints List Dict]])
 
 
 (defn run-coroutine [coro]
@@ -48,6 +48,11 @@
   (assert (= (get annotations "x") int))
   (assert (= (get annotations "z") bool)))
 
+(defn test-of []
+  (assert (= (of str) str))
+  (assert (= (of List int) (get List int)))
+  (assert (= (of Dict str str) (get Dict (, str str)))))
+
 (defn test-pep-487 []
   (defclass QuestBase []
     (defn --init-subclass-- [cls swallow &kwargs kwargs]

From 0579561b83047af8bb76ba1a7fe2f738184e6051 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Mon, 7 Oct 2019 19:20:54 -0500
Subject: [PATCH 213/223] Drop clint for colors in favor of colorama

Closes #1820.
---
 NEWS.rst                    |  5 ++++
 docs/language/internals.rst |  6 ++--
 hy/cmdline.py               |  3 ++
 hy/errors.py                | 19 +++++-------
 hy/models.py                | 60 +++++++++++++++++++++++--------------
 setup.py                    |  2 +-
 tests/test_models.py        | 13 ++++----
 7 files changed, 64 insertions(+), 44 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index 3c7db04..592a15d 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -31,6 +31,11 @@ Bug Fixes
 * `hy2py` can now handle format strings.
 * Fixed crashes from inaccessible history files.
 
+Misc. Improvements
+------------------------------
+* Drop the use of the long-abandoned `clint <https://github.com/kennethreitz/clint>`_ library
+  for colors. `colorama <https://github.com/tartley/colorama>`_ is now used instead.
+
 0.17.0
 ==============================
 
diff --git a/docs/language/internals.rst b/docs/language/internals.rst
index 2fb29bc..ec52c5f 100644
--- a/docs/language/internals.rst
+++ b/docs/language/internals.rst
@@ -44,9 +44,9 @@ If this is causing issues,
 it can be turned off globally by setting ``hy.models.PRETTY`` to ``False``,
 or temporarily by using the ``hy.models.pretty`` context manager.
 
-Hy also attempts to color pretty reprs using ``clint.textui.colored``.
-This module has a flag to disable coloring,
-and a method ``clean`` to strip colored strings of their color tags.
+Hy also attempts to color pretty reprs and errors using ``colorama``. These can
+be turned off globally by setting ``hy.models.COLORED`` and ``hy.errors.COLORED``,
+respectively, to ``False``.
 
 .. _hysequence:
 
diff --git a/hy/cmdline.py b/hy/cmdline.py
index 4c0c1fa..9ccb644 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -4,6 +4,9 @@
 
 from __future__ import print_function
 
+import colorama
+colorama.init()
+
 import argparse
 import code
 import ast
diff --git a/hy/errors.py b/hy/errors.py
index 0b7619e..c19cf6c 100644
--- a/hy/errors.py
+++ b/hy/errors.py
@@ -9,14 +9,13 @@ import traceback
 import pkgutil
 
 from functools import reduce
+from colorama import Fore
 from contextlib import contextmanager
 from hy import _initialize_env_var
 
-from clint.textui import colored
-
 _hy_filter_internal_errors = _initialize_env_var('HY_FILTER_INTERNAL_ERRORS',
                                                  True)
-_hy_colored_errors = _initialize_env_var('HY_COLORED_ERRORS', False)
+COLORED = _initialize_env_var('HY_COLORED_ERRORS', False)
 
 
 class HyError(Exception):
@@ -108,15 +107,12 @@ class HyLanguageError(HyError):
         """Provide an exception message that includes SyntaxError-like source
         line information when available.
         """
-        global _hy_colored_errors
-
         # Syntax errors are special and annotate the traceback (instead of what
         # we would do in the message that follows the traceback).
         if isinstance(self, SyntaxError):
             return super(HyLanguageError, self).__str__()
-
         # When there isn't extra source information, use the normal message.
-        if not isinstance(self, SyntaxError) and not self.text:
+        elif not self.text:
             return super(HyLanguageError, self).__str__()
 
         # Re-purpose Python's builtin syntax error formatting.
@@ -142,15 +138,14 @@ class HyLanguageError(HyError):
             output[arrow_idx] = '{}{}^\n'.format(output[arrow_idx].rstrip('\n'),
                                                  '-' * (self.arrow_offset - 1))
 
-        if _hy_colored_errors:
-            from clint.textui import colored
-            output[msg_idx:] = [colored.yellow(o) for o in output[msg_idx:]]
+        if COLORED:
+            output[msg_idx:] = [Fore.YELLOW + o + Fore.RESET for o in output[msg_idx:]]
             if arrow_idx:
-                output[arrow_idx] = colored.green(output[arrow_idx])
+                output[arrow_idx] = Fore.GREEN + output[arrow_idx] + Fore.RESET
             for idx, line in enumerate(output[::msg_idx]):
                 if line.strip().startswith(
                         'File "{}", line'.format(self.filename)):
-                    output[idx] = colored.red(line)
+                    output[idx] = Fore.RED + line + Fore.RESET
 
         # This resulting string will come after a "<class-name>:" prompt, so
         # put it down a line.
diff --git a/hy/models.py b/hy/models.py
index 55d0b79..6f42815 100644
--- a/hy/models.py
+++ b/hy/models.py
@@ -8,10 +8,10 @@ from math import isnan, isinf
 from hy import _initialize_env_var
 from hy.errors import HyWrapperError
 from fractions import Fraction
-from clint.textui import colored
+from colorama import Fore
 
 PRETTY = True
-_hy_colored_ast_objects = _initialize_env_var('HY_COLORED_AST_OBJECTS', False)
+COLORED = _initialize_env_var('HY_COLORED_AST_OBJECTS', False)
 
 
 @contextmanager
@@ -28,6 +28,18 @@ def pretty(pretty=True):
         PRETTY = old
 
 
+class _ColoredModel:
+    """
+    Mixin that provides a helper function for models that have color.
+    """
+
+    def _colored(self, text):
+        if COLORED:
+            return self.color + text + Fore.RESET
+        else:
+            return text
+
+
 class HyObject(object):
     """
     Generic Hy Object model. This is helpful to inject things into all the
@@ -243,7 +255,7 @@ class HyComplex(HyObject, complex):
 _wrappers[complex] = HyComplex
 
 
-class HySequence(HyObject, tuple):
+class HySequence(HyObject, tuple, _ColoredModel):
     """
     An abstract type for sequence-like models to inherit from.
     """
@@ -276,21 +288,25 @@ class HySequence(HyObject, tuple):
         return str(self) if PRETTY else super(HySequence, self).__repr__()
 
     def __str__(self):
-        global _hy_colored_ast_objects
         with pretty():
-            c = self.color if _hy_colored_ast_objects else str
             if self:
-                return ("{}{}\n  {}{}").format(
-                    c(self.__class__.__name__),
-                    c("(["),
-                    (c(",") + "\n  ").join([repr_indent(e) for e in self]),
-                    c("])"))
+                return self._colored("{}{}\n  {}{}".format(
+                    self._colored(self.__class__.__name__),
+                    self._colored("(["),
+                    self._colored(",\n  ").join(map(repr_indent, self)),
+                    self._colored("])"),
+                ))
+                return self._colored("{}([\n  {}])".format(
+                    self.__class__.__name__,
+                    ','.join(repr_indent(e) for e in self),
+                ))
             else:
-                return '' + c(self.__class__.__name__ + "()")
+                return self._colored(self.__class__.__name__ + "()")
 
 
 class HyList(HySequence):
-    color = staticmethod(colored.cyan)
+    color = Fore.CYAN
+
 
 def recwrap(f):
     return lambda l: f(wrap_value(x) for x in l)
@@ -300,16 +316,14 @@ _wrappers[list] = recwrap(HyList)
 _wrappers[tuple] = recwrap(HyList)
 
 
-class HyDict(HySequence):
+class HyDict(HySequence, _ColoredModel):
     """
     HyDict (just a representation of a dict)
     """
-    color = staticmethod(colored.green)
+    color = Fore.GREEN
 
     def __str__(self):
-        global _hy_colored_ast_objects
         with pretty():
-            g = self.color if _hy_colored_ast_objects else str
             if self:
                 pairs = []
                 for k, v in zip(self[::2],self[1::2]):
@@ -317,14 +331,16 @@ class HyDict(HySequence):
                     pairs.append(
                         ("{0}{c}\n  {1}\n  "
                          if '\n' in k+v
-                         else "{0}{c} {1}").format(k, v, c=g(',')))
+                         else "{0}{c} {1}").format(k, v, c=self._colored(',')))
                 if len(self) % 2 == 1:
                     pairs.append("{}  {}\n".format(
-                        repr_indent(self[-1]), g("# odd")))
+                        repr_indent(self[-1]), self._colored("# odd")))
                 return "{}\n  {}{}".format(
-                    g("HyDict(["), ("{c}\n  ".format(c=g(',')).join(pairs)), g("])"))
+                    self._colored("HyDict(["),
+                    "{c}\n  ".format(c=self._colored(',')).join(pairs),
+                    self._colored("])"))
             else:
-                return '' + g("HyDict()")
+                return self._colored("HyDict()")
 
     def keys(self):
         return list(self[0::2])
@@ -343,7 +359,7 @@ class HyExpression(HySequence):
     """
     Hy S-Expression. Basically just a list.
     """
-    color = staticmethod(colored.yellow)
+    color = Fore.YELLOW
 
 _wrappers[HyExpression] = recwrap(HyExpression)
 _wrappers[Fraction] = lambda e: HyExpression(
@@ -354,7 +370,7 @@ class HySet(HySequence):
     """
     Hy set (just a representation of a set)
     """
-    color = staticmethod(colored.red)
+    color = Fore.RED
 
 _wrappers[HySet] = recwrap(HySet)
 _wrappers[set] = recwrap(HySet)
diff --git a/setup.py b/setup.py
index 9499601..eaebf95 100755
--- a/setup.py
+++ b/setup.py
@@ -35,7 +35,7 @@ install_requires = [
     'rply>=0.7.7',
     'astor>=0.8',
     'funcparserlib>=0.3.6',
-    'clint>=0.4']
+    'colorama']
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 
diff --git a/tests/test_models.py b/tests/test_models.py
index 52d4518..3670974 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -4,10 +4,11 @@
 
 import copy
 import hy
-from clint.textui.colored import clean
 from hy.models import (wrap_value, replace_hy_obj, HyString, HyInteger, HyList,
                        HyDict, HySet, HyExpression, HyComplex, HyFloat, pretty)
 
+hy.models.COLORED = False
+
 
 def test_wrap_int():
     """ Test conversion of integers."""
@@ -182,13 +183,13 @@ def test_compound_model_repr():
             assert eval(repr(model([1, 2, 3]))) == model([1, 2, 3])
         for k, v in PRETTY_STRINGS.items():
             # `str` should be pretty, even under `pretty(False)`.
-            assert clean(str(hy.read_str(k))) == v
+            assert str(hy.read_str(k)) == v
         for k in PRETTY_STRINGS.keys():
             assert eval(repr(hy.read_str(k))) == hy.read_str(k)
     with pretty(True):
         for model in HY_LIST_MODELS:
-            assert eval(clean(repr(model()))).__class__ is model
-            assert eval(clean(repr(model([1, 2])))) == model([1, 2])
-            assert eval(clean(repr(model([1, 2, 3])))) == model([1, 2, 3])
+            assert eval(repr(model())).__class__ is model
+            assert eval(repr(model([1, 2]))) == model([1, 2])
+            assert eval(repr(model([1, 2, 3]))) == model([1, 2, 3])
         for k, v in PRETTY_STRINGS.items():
-            assert clean(repr(hy.read_str(k))) == v
+            assert repr(hy.read_str(k)) == v

From 2f86801a145bd16e00ea444291266c546687ef81 Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Mon, 7 Oct 2019 19:17:36 -0500
Subject: [PATCH 214/223] Add documentation for annotations and `of`

---
 docs/language/api.rst | 69 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/docs/language/api.rst b/docs/language/api.rst
index 496f3da..9592d48 100644
--- a/docs/language/api.rst
+++ b/docs/language/api.rst
@@ -8,6 +8,48 @@ Hy features a number of special forms that are used to help generate
 correct Python AST. The following are "special" forms, which may have
 behavior that's slightly unexpected in some situations.
 
+^
+-
+
+The ``^`` symbol is used to denote annotations in three different contexts:
+
+- Standalone variable annotations.
+- Variable annotations in a setv call.
+- Function argument annotations.
+
+They implement `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_ and
+`PEP 3107 <https://www.python.org/dev/peps/pep-3107/>`_.
+
+Here is some example syntax of all three usages:
+
+.. code-block:: clj
+
+  ; Annotate the variable x as an int (equivalent to `x: int`).
+  (^int x)
+  ; Can annotate with expressions if needed (equivalent to `y: f(x)`).
+  (^(f x) y)
+
+  ; Annotations with an assignment: each annotation (int, str) covers the term that
+  ; immediately follows.
+  ; Equivalent to: x: int = 1; y = 2; z: str = 3
+  (setv ^int x 1 y 2 ^str z 3)
+
+  ; Annotate a as an int, c as an int, and b as a str.
+  ; Equivalent to: def func(a: int, b: str = None, c: int = 1): ...
+  (defn func [^int a &optional ^str b ^int [c 1]] ...)
+
+The rules are:
+
+- The value to annotate with is the value that immediately follows the caret.
+- There must be no space between the caret and the value to annotate, otherwise it will be
+  interpreted as a bitwise XOR like the Python operator.
+- The annotation always comes (and is evaluated) *before* the value being annotated. This is
+  unlike Python, where it comes and is evaluated *after* the value being annotated.
+
+Note that variable annotations are only supported on Python 3.6+.
+
+For annotating items with generic types, the of_ macro will likely be of use.
+
 .
 -
 
@@ -1408,6 +1450,33 @@ parameter will be returned.
 
 .. _py-specialform:
 
+of
+--
+
+``of`` is an alias for get, but with special semantics designed for handling PEP 484's generic
+types.
+
+``of`` has three forms:
+
+- ``(of T)`` will simply become ``T``.
+- ``(of T x)`` will become ``(get T x)``.
+- ``(of T x y ...)`` (where the ``...`` represents zero or more arguments) will become
+  ``(get T (, x y ...))``.
+
+For instance:
+
+.. code-block:: clj
+
+  (of str)  ; => str
+
+  (of List int)  ; => List[int]
+  (of Set int)   ; => Set[int]
+
+  (of Dict str str)   ; => Dict[str, str]
+  (of Tuple str int)  ; => Tuple[str, int]
+
+  (of Callable [int str] str)  ; => Callable[[int, str], str]
+
 py
 --
 

From beb21d384c48e352d2217e6ffdd46d2fdb6bf92e Mon Sep 17 00:00:00 2001
From: Ryan Gonzalez <rymg19@gmail.com>
Date: Thu, 10 Oct 2019 17:08:53 -0500
Subject: [PATCH 215/223] Fix a unit test bug on slim Python Docker images

If HyASTCompiler is given a string, it imports it and uses it as the
execution environment. However, the unit tests gave HyASTCompiler the
string 'test', assuming it would create a new test module, when in
reality it would import CPython's test module that is designed for
internal use. Slim Docker images don't include this module, therefore
the tests would fail to run.
---
 NEWS.rst                         | 2 ++
 tests/compilers/test_compiler.py | 5 +++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index e5b1d69..4c59c34 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -31,6 +31,8 @@ Bug Fixes
 * Fixed the expression of a while loop that contains statements being compiled twice.
 * `hy2py` can now handle format strings.
 * Fixed crashes from inaccessible history files.
+* The unit tests no longer unintentionally import the internal Python module "test".
+  This allows them to pass when run inside the "slim" Python Docker images.
 
 0.17.0
 ==============================
diff --git a/tests/compilers/test_compiler.py b/tests/compilers/test_compiler.py
index 9648c8f..879bd58 100644
--- a/tests/compilers/test_compiler.py
+++ b/tests/compilers/test_compiler.py
@@ -6,6 +6,7 @@ import ast
 
 from hy import compiler
 from hy.models import HyExpression, HyList, HySymbol, HyInteger
+import types
 
 
 def make_expression(*args):
@@ -25,7 +26,7 @@ def test_compiler_bare_names():
                         HySymbol("a"),
                         HySymbol("b"),
                         HySymbol("c"))
-    ret = compiler.HyASTCompiler('test').compile(e)
+    ret = compiler.HyASTCompiler(types.ModuleType('test')).compile(e)
 
     # We expect two statements and a final expr.
 
@@ -54,7 +55,7 @@ def test_compiler_yield_return():
                         HyExpression([HySymbol("+"),
                                       HyInteger(1),
                                       HyInteger(1)]))
-    ret = compiler.HyASTCompiler('test').compile_atom(e)
+    ret = compiler.HyASTCompiler(types.ModuleType('test')).compile_atom(e)
 
     assert len(ret.stmts) == 1
     stmt, = ret.stmts

From 7af169f089b773836b9587b5bcebcb20bf0bc023 Mon Sep 17 00:00:00 2001
From: Noah Snelson <noah.snelson@protonmail.com>
Date: Sun, 13 Oct 2019 22:50:47 -0700
Subject: [PATCH 216/223] Add HyHelper object to override builtins.help, fixes
 REPL prompt

---
 hy/cmdline.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/hy/cmdline.py b/hy/cmdline.py
index 4c0c1fa..02b0609 100644
--- a/hy/cmdline.py
+++ b/hy/cmdline.py
@@ -59,9 +59,19 @@ class HyQuitter(object):
             pass
         raise SystemExit(code)
 
+class HyHelper(object):
+    def __repr__(self):
+        return ("Use (help) for interactive help, or (help object) for help "
+                "about object.")
+
+    def __call__(self, *args, **kwds):
+        import pydoc
+        return pydoc.help(*args, **kwds)
+
 
 builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
+builtins.help = HyHelper()
 
 @contextmanager
 def extend_linecache(add_cmdline_cache):

From 0965966cdebc51fc0bc1d793e1b2b2071b7ca901 Mon Sep 17 00:00:00 2001
From: Noah Snelson <noah.snelson@protonmail.com>
Date: Sun, 13 Oct 2019 22:52:10 -0700
Subject: [PATCH 217/223] Added name to AUTHORS file.

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b987a9b..6ae7875 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -93,3 +93,4 @@
 * Andrew R. M. <nixy@nixy.moe>
 * Tristan de Cacqueray <tdecacqu@redhat.com>
 * Sören Tempel <soeren@soeren-tempel.net>
+* Noah Snelson <noah.snelson@protonmail.com>

From 0f3d256ebfe9e2f0adaecc96b8840d8d2328dfca Mon Sep 17 00:00:00 2001
From: Adam Porter <adam@alphapapa.net>
Date: Wed, 30 Oct 2019 08:45:17 -0500
Subject: [PATCH 218/223] Add: parse-args function

Closes #1719.
---
 NEWS.rst                   |  1 +
 docs/language/core.rst     | 23 +++++++++++++++++++++++
 hy/core/language.hy        | 15 ++++++++++++++-
 tests/native_tests/core.hy |  9 +++++++++
 4 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/NEWS.rst b/NEWS.rst
index 40bd19b..2053b1f 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -18,6 +18,7 @@ New Features
 * All augmented assignment operators (except `%=` and `^=`) now allow
   more than two arguments.
 * PEP 3107 and PEP 526 function and variable annotations are now supported.
+* Added function ``parse-args`` which parses arguments with ``argparse``.
 
 Other Breaking Changes
 ------------------------------
diff --git a/docs/language/core.rst b/docs/language/core.rst
index cabb420..6cb8c4d 100644
--- a/docs/language/core.rst
+++ b/docs/language/core.rst
@@ -786,6 +786,29 @@ Returns ``True`` if *x* is odd. Raises ``TypeError`` if
    => (odd? 0)
    False
 
+.. _parse-args:
+
+parse-args
+----------
+
+Usage: ``(parse-args spec &optional args &kwargs parser-args)``
+
+Return arguments namespace parsed from *args* or ``sys.argv`` with
+:py:meth:`argparse.ArgumentParser.parse_args` according to *spec*.
+
+*spec* should be a list of arguments which will be passed to repeated
+calls to :py:meth:`argparse.ArgumentParser.add_argument`.  *parser-args*
+may be a list of keyword arguments to pass to the
+:py:class:`argparse.ArgumentParser` constructor.
+
+.. code-block:: hy
+
+    => (parse-args [["strings" :nargs "+" :help "Strings"]
+                    ["-n" "--numbers" :action "append" :type 'int :help "Numbers"]]
+                   ["a" "b" "-n" "1" "-n" "2"]
+                   :description "Parse strings and numbers from args")
+    Namespace(numbers=[1, 2], strings=['a', 'b'])
+
 .. _partition-fn:
 
 partition
diff --git a/hy/core/language.hy b/hy/core/language.hy
index 80cba83..fa82185 100644
--- a/hy/core/language.hy
+++ b/hy/core/language.hy
@@ -391,6 +391,19 @@ Even objects with the __name__ magic will work."
     False
     (or a b)))
 
+(defn parse-args [spec &optional args &kwargs parser-args]
+  "Return arguments namespace parsed from `args` or `sys.argv` with `argparse.ArgumentParser.parse-args` according to `spec`.
+
+`spec` should be a list of arguments to pass to repeated calls to
+`argparse.ArgumentParser.add-argument`.  `parser-args` may be a list
+of keyword arguments to pass to the `argparse.ArgumentParser`
+constructor."
+  (import argparse)
+  (setv parser (argparse.ArgumentParser #** parser-args))
+  (for [arg spec]
+    (eval `(.add-argument parser ~@arg)))
+  (.parse-args parser args))
+
 (setv EXPORTS
   '[*map accumulate butlast calling-module calling-module-name chain coll?
     combinations comp complement compress constantly count cycle dec distinct
@@ -399,6 +412,6 @@ Even objects with the __name__ magic will work."
     integer? integer-char? interleave interpose islice iterable?
     iterate iterator? juxt keyword keyword? last list? macroexpand
     macroexpand-1 mangle merge-with multicombinations name neg? none? nth
-    numeric? odd? partition permutations pos? product read read-str
+    numeric? odd? parse-args partition permutations pos? product read read-str
     remove repeat repeatedly rest reduce second some string? symbol?
     take take-nth take-while tuple? unmangle xor tee zero? zip-longest])
diff --git a/tests/native_tests/core.hy b/tests/native_tests/core.hy
index b961f78..e5aad68 100644
--- a/tests/native_tests/core.hy
+++ b/tests/native_tests/core.hy
@@ -478,6 +478,15 @@ result['y in globals'] = 'y' in globals()")
   (assert-false (odd? 0))
   (assert-requires-num odd?))
 
+(defn test-parse-args []
+  "NATIVE: testing the parse-args function"
+  (setv parsed-args (parse-args [["strings" :nargs "+" :help "Strings"]
+                                 ["-n" "--numbers" :action "append" :type 'int :help "Numbers"]]
+                                ["a" "b" "-n" "1" "-n" "2"]
+                                :description "Parse strings and numbers from args"))
+  (assert-equal parsed-args.strings ["a" "b"])
+  (assert-equal parsed-args.numbers [1 2]))
+
 (defn test-partition []
   "NATIVE: testing the partition function"
   (setv ten (range 10))

From 825dfe3eeb19db868e5f1c69fc0b4be5dfc8b8ed Mon Sep 17 00:00:00 2001
From: Adam Porter <adam@alphapapa.net>
Date: Wed, 30 Oct 2019 08:53:55 -0500
Subject: [PATCH 219/223] AUTHORS: Add Adam Porter

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 6ae7875..54e05bb 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -94,3 +94,4 @@
 * Tristan de Cacqueray <tdecacqu@redhat.com>
 * Sören Tempel <soeren@soeren-tempel.net>
 * Noah Snelson <noah.snelson@protonmail.com>
+* Adam Porter <adam@alphapapa.net>

From cb256fd618ae26708e106c5529aab4ea1a01baf8 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Tue, 12 Nov 2019 16:27:07 -0500
Subject: [PATCH 220/223] Document gotcha with unintended recursion in `let`

---
 docs/contrib/walk.rst | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/contrib/walk.rst b/docs/contrib/walk.rst
index e6d333e..8fea960 100644
--- a/docs/contrib/walk.rst
+++ b/docs/contrib/walk.rst
@@ -240,7 +240,8 @@ The ``let`` macro takes two parameters: a list defining *variables*
 and the *body* which gets executed. *variables* is a vector of
 variable and value pairs.
 
-``let`` executes the variable assignments one-by-one, in the order written.
+Like the ``let*`` of many other Lisps, ``let`` executes the variable
+assignments one-by-one, in the order written::
 
 .. code-block:: hy
 
@@ -249,4 +250,8 @@ variable and value pairs.
     ...  (print x y))
     5 6
 
+Unlike them, however, each ``(let …)`` form uses only one
+namespace for all its assignments. Thus, ``(let [x 1  x (fn [] x)]
+(x))`` returns a function object, not 1 as you might expect.
+
 It is an error to use a let-bound name in a ``global`` or ``nonlocal`` form.

From a50b381e6f133d92a259905a2a54a52a90c04ae3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Sun, 17 Nov 2019 10:14:04 -0500
Subject: [PATCH 221/223] Use the released version of Python 3.8 on Travis
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Gábor Lipták <gliptak@gmail.com>
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f7fe97b..31862a6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ python:
   - "3.5"
   - "3.6"
   - "3.7"
-  - 3.8-dev
+  - "3.8"
   - pypy3.5-6.0
 install:
   - pip install -r requirements-travis.txt

From 832253a0c051fef022f9fb13a9d3c5f41f09b8bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= <gliptak@gmail.com>
Date: Sun, 17 Nov 2019 10:28:37 -0500
Subject: [PATCH 222/223] =?UTF-8?q?Add=20G=C3=A1bor=20Lipt=C3=A1k=20to=20A?=
 =?UTF-8?q?UTHORS?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Gábor Lipták <gliptak@gmail.com>
---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 54e05bb..bdae6fa 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -95,3 +95,4 @@
 * Sören Tempel <soeren@soeren-tempel.net>
 * Noah Snelson <noah.snelson@protonmail.com>
 * Adam Porter <adam@alphapapa.net>
+* Gábor Lipták <gliptak@gmail.com>

From 8872f0b44c5f51e1206b32d960706e5772a28e94 Mon Sep 17 00:00:00 2001
From: Kodi Arfer <git@arfer.net>
Date: Mon, 18 Nov 2019 20:06:17 -0500
Subject: [PATCH 223/223] Remove hy.contrib.multi

---
 NEWS.rst                            |   2 +
 docs/contrib/index.rst              |   1 -
 docs/contrib/multi.rst              | 110 -------------------------
 hy/contrib/multi.hy                 |  89 --------------------
 tests/native_tests/contrib/multi.hy | 123 ----------------------------
 5 files changed, 2 insertions(+), 323 deletions(-)
 delete mode 100644 docs/contrib/multi.rst
 delete mode 100644 hy/contrib/multi.hy
 delete mode 100644 tests/native_tests/contrib/multi.hy

diff --git a/NEWS.rst b/NEWS.rst
index 2053b1f..a20bd14 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -10,6 +10,8 @@ Removals
   and `defn` instead.
 * Literal keywords are no longer parsed differently in calls to functions
   with certain names.
+* `hy.contrib.multi` has been removed. Use ``cond`` or the PyPI package
+  ``multipledispatch`` instead.
 
 New Features
 ------------------------------
diff --git a/docs/contrib/index.rst b/docs/contrib/index.rst
index 917e12b..32936e0 100644
--- a/docs/contrib/index.rst
+++ b/docs/contrib/index.rst
@@ -12,7 +12,6 @@ Contents:
    :maxdepth: 3
 
    loop
-   multi
    profile
    sequences
    walk
diff --git a/docs/contrib/multi.rst b/docs/contrib/multi.rst
deleted file mode 100644
index b0a035f..0000000
--- a/docs/contrib/multi.rst
+++ /dev/null
@@ -1,110 +0,0 @@
-========
-defmulti
-========
-
-defn
-----
-.. versionadded:: 0.10.0
-
-``defn`` lets you arity-overload a function by the given number of
-args and/or kwargs. This version of ``defn`` works with regular syntax and
-with the arity overloaded one. Inspired by Clojures take on ``defn``.
-
-.. code-block:: clj
-
-    => (require [hy.contrib.multi [defn]])
-    => (defn fun
-    ...  ([a] "a")
-    ...  ([a b] "a b")
-    ...  ([a b c] "a b c"))
-
-    => (fun 1)
-    "a"
-    => (fun 1 2)
-    "a b"
-    => (fun 1 2 3)
-    "a b c"
-
-    => (defn add [a b]
-    ...  (+ a b))
-    => (add 1 2)
-    3
-
-
-defmulti
---------
-
-.. versionadded:: 0.12.0
-
-``defmulti``, ``defmethod`` and ``default-method`` lets you define
-multimethods where a dispatching function is used to select between different
-implementations of the function. Inspired by Clojure's multimethod and based
-on the code by `Adam Bard`_.
-
-.. code-block:: clj
-
-    => (require [hy.contrib.multi [defmulti defmethod default-method]])
-    => (defmulti area [shape]
-    ...  "calculate area of a shape"
-    ...  (:type shape))
-
-    => (defmethod area "square" [square]
-    ...  (* (:width square)
-    ...     (:height square)))
-
-    => (defmethod area "circle" [circle]
-    ...  (* (** (:radius circle) 2)
-    ...     3.14))
-
-    => (default-method area [shape]
-    ...  0)
-
-    => (area {:type "circle" :radius 0.5})
-    0.785
-
-    => (area {:type "square" :width 2 :height 2})
-    4
-
-    => (area {:type "non-euclid rhomboid"})
-    0
-
-``defmulti`` is used to define the initial multimethod with name, signature
-and code that selects between different implementations. In the example,
-multimethod expects a single input that is type of dictionary and contains
-at least key :type. The value that corresponds to this key is returned and
-is used to selected between different implementations.
-
-``defmethod`` defines a possible implementation for multimethod. It works
-otherwise in the same way as ``defn``, but has an extra parameters
-for specifying multimethod and which calls are routed to this specific
-implementation. In the example, shapes with "square" as :type are routed to
-first function and shapes with "circle" as :type are routed to second
-function.
-
-``default-method`` specifies default implementation for multimethod that is
-called when no other implementation matches.
-
-Interfaces of multimethod and different implementation don't have to be
-exactly identical, as long as they're compatible enough. In practice this
-means that multimethod should accept the broadest range of parameters and
-different implementations can narrow them down.
-
-.. code-block:: clj
-
-    => (require [hy.contrib.multi [defmulti defmethod]])
-    => (defmulti fun [&rest args]
-    ...  (len args))
-
-    => (defmethod fun 1 [a]
-    ...  a)
-
-    => (defmethod fun 2 [a b]
-    ...  (+ a b))
-
-    => (fun 1)
-    1
-
-    => (fun 1 2)
-    3
-
-.. _Adam Bard: https://adambard.com/blog/implementing-multimethods-in-python/
diff --git a/hy/contrib/multi.hy b/hy/contrib/multi.hy
deleted file mode 100644
index c84adf9..0000000
--- a/hy/contrib/multi.hy
+++ /dev/null
@@ -1,89 +0,0 @@
-;; Hy Arity-overloading
-;; Copyright 2019 the authors.
-;; This file is part of Hy, which is free software licensed under the Expat
-;; license. See the LICENSE.
-
-(import [collections [defaultdict]]
-        [hy [HyExpression HyList HyString]])
-
-(defclass MultiDispatch [object]
-
-  (setv _fns (defaultdict dict))
-
-  (defn __init__ [self f]
-    (setv self.f f)
-    (setv self.__doc__ f.__doc__)
-    (unless (in f.__name__ (.keys (get self._fns f.__module__)))
-      (setv (get self._fns f.__module__ f.__name__) {}))
-    (setv values f.__code__.co_varnames)
-    (setv (get self._fns f.__module__ f.__name__ values) f))
-
-  (defn fn? [self v args kwargs]
-    "Compare the given (checked fn) to the called fn"
-    (setv com (+ (list args) (list (.keys kwargs))))
-    (and
-      (= (len com) (len v))
-      (.issubset (frozenset (.keys kwargs)) com)))
-
-  (defn __call__ [self &rest args &kwargs kwargs]
-    (setv func None)
-    (for [[i f] (.items (get self._fns self.f.__module__ self.f.__name__))]
-      (when (.fn? self i args kwargs)
-        (setv func f)
-        (break)))
-    (if func
-      (func #* args #** kwargs)
-      (raise (TypeError "No matching functions with this signature")))))
-
-(defn multi-decorator [dispatch-fn]
-  (setv inner (fn [&rest args &kwargs kwargs]
-                (setv dispatch-key (dispatch-fn #* args #** kwargs))
-                (if (in dispatch-key inner.--multi--)
-                  ((get inner.--multi-- dispatch-key) #* args #** kwargs)
-                  (inner.--multi-default-- #* args #** kwargs))))
-  (setv inner.--multi-- {})
-  (setv inner.--doc-- dispatch-fn.--doc--)
-  (setv inner.--multi-default-- (fn [&rest args &kwargs kwargs] None))
-  inner)
-
-(defn method-decorator [dispatch-fn &optional [dispatch-key None]]
-  (setv apply-decorator
-        (fn [func]
-          (if (is dispatch-key None)
-            (setv dispatch-fn.--multi-default-- func)
-           (assoc dispatch-fn.--multi-- dispatch-key func))
-          dispatch-fn))
-  apply-decorator)
- 
-(defmacro defmulti [name params &rest body]
-  `(do (import [hy.contrib.multi [multi-decorator]])
-       (with-decorator multi-decorator
-         (defn ~name ~params ~@body))))
-
-(defmacro defmethod [name multi-key params &rest body]
-  `(do (import [hy.contrib.multi [method-decorator]])
-       (with-decorator (method-decorator ~name ~multi-key)
-         (defn ~name ~params ~@body))))
-
-(defmacro default-method [name params &rest body]
-  `(do (import [hy.contrib.multi [method-decorator]])
-       (with-decorator (method-decorator ~name)
-         (defn ~name ~params ~@body))))
-
-(defmacro defn [name &rest bodies]
-  (setv arity-overloaded? (fn [bodies]
-                            (if (isinstance (first bodies) HyString)
-                                (arity-overloaded? (rest bodies))
-                                (isinstance (first bodies) HyExpression))))
-
-  (if (arity-overloaded? bodies)
-    (do
-     (setv comment (HyString))
-     (if (= (type (first bodies)) HyString)
-       (setv [comment #* bodies] bodies))
-     (+ '(do (import [hy.contrib.multi [MultiDispatch]])) (lfor
-       [let-binds #* body] bodies
-       `(with-decorator MultiDispatch (defn ~name ~let-binds ~comment ~@body)))))
-    (do
-     (setv [lambda-list #* body] bodies)
-     `(setv ~name (fn* ~lambda-list ~@body)))))
diff --git a/tests/native_tests/contrib/multi.hy b/tests/native_tests/contrib/multi.hy
deleted file mode 100644
index ab4d85c..0000000
--- a/tests/native_tests/contrib/multi.hy
+++ /dev/null
@@ -1,123 +0,0 @@
-;; Copyright 2019 the authors.
-;; This file is part of Hy, which is free software licensed under the Expat
-;; license. See the LICENSE.
-
-(require [hy.contrib.multi [defmulti defmethod default-method defn]])
-(import pytest)
-
-(defn test-different-signatures []
-  "NATIVE: Test multimethods with different signatures"
-  (defmulti fun [&rest args]
-    (len args))
-
-  (defmethod fun 0 []
-    "Hello!")
-
-  (defmethod fun 1 [a]
-    a)
-
-  (defmethod fun 2 [a b]
-    "a b")
-
-  (defmethod fun 3 [a b c]
-    "a b c")
-
-  (assert (= (fun) "Hello!"))
-  (assert (= (fun "a") "a"))
-  (assert (= (fun "a" "b") "a b"))
-  (assert (= (fun "a" "b" "c") "a b c")))
-
-(defn test-different-signatures-defn []
-  "NATIVE: Test defn with different signatures"
-  (defn f1
-    ([] "")
-    ([a] "a")
-    ([a b] "a b"))
-
-  (assert (= (f1) ""))
-  (assert (= (f1 "a") "a"))
-  (assert (= (f1 "a" "b") "a b"))
-  (with [(pytest.raises TypeError)]
-    (f1 "a" "b" "c")))
-
-(defn test-basic-dispatch []
-  "NATIVE: Test basic dispatch"
-  (defmulti area [shape]
-    (:type shape))
-  
-  (defmethod area "square" [square]
-    (* (:width square)
-       (:height square)))
-  
-  (defmethod area "circle" [circle]
-    (* (** (:radius circle) 2) 
-       3.14))
-
-  (default-method area [shape]
-    0)
-
-  (assert (< 0.784 (area {:type "circle" :radius 0.5}) 0.786))
-  (assert (= (area {:type "square" :width 2 :height 2})) 4)
-  (assert (= (area {:type "non-euclid rhomboid"}) 0)))
-
-(defn test-docs []
-  "NATIVE: Test if docs are properly handled"
-  (defmulti fun [a b]
-    "docs"
-    a)
-
-  (defmethod fun "foo" [a b]
-    "foo was called")
-
-  (defmethod fun "bar" [a b]
-    "bar was called")
-  
-  (assert (= fun.--doc-- "docs")))
-
-(defn test-kwargs-handling []
-  "NATIVE: Test handling of kwargs with multimethods"
-  (defmulti fun [&kwargs kwargs]
-    (get kwargs "type"))
-
-  (defmethod fun "foo" [&kwargs kwargs]
-    "foo was called")
-
-  (defmethod fun "bar" [&kwargs kwargs]
-    "bar was called")
-
-  (assert (= (fun :type "foo" :extra "extra") "foo was called")))
-
-(defn test-basic-multi []
-  "NATIVE: Test a basic arity overloaded defn"
-  (defn f2
-    ([] "Hello!")
-    ([a] a)
-    ([a b] "a b")
-    ([a b c] "a b c"))
-
-  (assert (= (f2) "Hello!"))
-  (assert (= (f2 "a") "a"))
-  (assert (= (f2 "a" "b") "a b"))
-  (assert (= (f2 "a" "b" "c") "a b c")))
-
-
-(defn test-kw-args []
-  "NATIVE: Test if kwargs are handled correctly for arity overloading"
-  (defn f3
-    ([a] a)
-    ([&optional [a "nop"] [b "p"]] (+ a b)))
-   
-  (assert (= (f3 1) 1))
-  (assert (= (f3 :a "t") "t"))
-  (assert (= (f3 "hello " :b "world") "hello world"))
-  (assert (= (f3 :a "hello " :b "world") "hello world")))
-
-
-(defn test-docs []
-  "NATIVE: Test if docs are properly handled for arity overloading"
-  (defn f4
-    "docs"
-    ([a] (print a))
-    ([a b] (print b)))
-  
-  (assert (= f4.--doc-- "docs")))