Merge pull request #1593 from Kodiologist/compiler-pattern-matching

Pattern-matching compiler

NEWS.rst

@@ -33,7 +33,9 @@ Other Breaking Changes
 * `hy-repr` uses registered functions instead of methods
 * `HyKeyword` no longer inherits from the string type and has been
   made into its own object type.
+* `HySymbol` no longer inherits from `HyString`.
 * `(except)` is no longer allowed. Use `(except [])` instead.
+* `(import [foo])` is no longer allowed. Use `(import foo)` instead.
 
 New Features
 ------------------------------

docs/language/internals.rst

@@ -119,13 +119,12 @@ the following order:
 HyString
 ~~~~~~~~
 
-``hy.models.HyString`` is the base class of string-equivalent Hy
+``hy.models.HyString`` represents string literals (including bracket strings),
-models. It also represents string literals (including bracket strings), which
+which compile down to unicode string literals in Python. ``HyStrings`` inherit
-compile down to unicode string literals in Python. ``HyStrings`` inherit
+unicode objects in Python 2, and string objects in Python 3 (and are therefore
-unicode objects in Python 2, and string objects in Python 3 (and are
+not encoding-dependent).
-therefore not encoding-dependent).
 
-``HyString`` based models are immutable.
+``HyString``\s are immutable.
 
 Hy literal strings can span multiple lines, and are considered by the
 parser as a single unit, respecting the Python escapes for unicode
@@ -163,8 +162,9 @@ valid numeric python literals will be turned into their Hy counterpart.
 HySymbol
 ~~~~~~~~
 
-``hy.models.HySymbol`` is the model used to represent symbols
+``hy.models.HySymbol`` is the model used to represent symbols in the Hy
-in the Hy language. It inherits :ref:`HyString`.
+language. Like ``HyString``, it inherits from ``str`` (or ``unicode`` on Python
+2).
 
 Symbols are :ref:`mangled <mangling>` when they are compiled
 to Python variable names.

hy/cmdline.py

@@ -186,8 +186,8 @@ def ideas_macro(ETname):
 
 """)])
 
-require("hy.cmdline", "__console__", all_macros=True)
+require("hy.cmdline", "__console__", assignments="ALL")
-require("hy.cmdline", "__main__", all_macros=True)
+require("hy.cmdline", "__main__", assignments="ALL")
 
 SIMPLE_TRACEBACKS = True
 

hy/completer.py

@@ -37,7 +37,7 @@ class Completer(object):
         if not isinstance(namespace, dict):
             raise TypeError('namespace must be a dictionary')
         self.namespace = namespace
-        self.path = [hy.compiler._compile_table,
+        self.path = [hy.compiler._special_form_compilers,
                      builtins.__dict__,
                      hy.macros._hy_macros[None],
                      namespace]

hy/contrib/walk.hy

@@ -72,9 +72,7 @@
 
 ;; TODO: move to hy.extra.reserved?
 (import hy)
-(setv special-forms (list-comp k
+(setv special-forms (list (.keys hy.compiler._special-form-compilers)))
-                               [k (.keys hy.compiler._compile-table)]
-                               (isinstance k hy._compat.string-types)))
 
 
 (defn lambda-list [form]

hy/extra/reserved.hy

@@ -18,6 +18,6 @@
       hy.core.shadow.EXPORTS
       (list (.keys (get hy.macros._hy_macros None)))
       keyword.kwlist
-      (list-comp k [k (.keys hy.compiler.-compile-table)]
+      (list (.keys hy.compiler._special_form_compilers))
-        (isinstance k hy._compat.string-types))))))))
+      (list hy.compiler._bad_roots)))))))
   _cache)

hy/macros.py

@@ -80,11 +80,10 @@ def tag(name):
     return _
 
 
-def require(source_module, target_module,
+def require(source_module, target_module, assignments, prefix=""):
-            all_macros=False, assignments={}, prefix=""):
     """Load macros from `source_module` in the namespace of
-    `target_module`. `assignments` maps old names to new names, but is
+    `target_module`. `assignments` maps old names to new names, or
-    ignored if `all_macros` is true. If `prefix` is nonempty, it is
+    should be the string "ALL". If `prefix` is nonempty, it is
     prepended to the name of each imported macro. (This means you get
     macros named things like "mymacromodule.mymacro", which looks like
     an attribute of a module, although it's actually just a symbol
@@ -97,17 +96,18 @@ def require(source_module, target_module,
     seen_names = set()
     if prefix:
         prefix += "."
-    assignments = {mangle(str_type(k)): v for k, v in assignments.items()}
+    if assignments != "ALL":
+        assignments = {mangle(str_type(k)): v for k, v in assignments}
 
     for d in _hy_macros, _hy_tag:
         for name, macro in d[source_module].items():
             seen_names.add(name)
-            if all_macros:
+            if assignments == "ALL":
                 d[target_module][mangle(prefix + name)] = macro
             elif name in assignments:
                 d[target_module][mangle(prefix + assignments[name])] = macro
 
-    if not all_macros:
+    if assignments != "ALL":
         unseen = frozenset(assignments.keys()).difference(seen_names)
         if unseen:
             raise ImportError("cannot require names: " + repr(list(unseen)))

hy/model_patterns.py

@@ -0,0 +1,76 @@
+# Copyright 2018 the authors.
+# This file is part of Hy, which is free software licensed under the Expat
+# license. See the LICENSE.
+
+"Parser combinators for pattern-matching Hy model trees."
+
+from hy.models import HyExpression, HySymbol, HyKeyword, HyString, HyList
+from funcparserlib.parser import (
+    some, skip, many, finished, a, Parser, NoParseError, State)
+from functools import reduce
+from itertools import repeat
+from operator import add
+from math import isinf
+
+FORM = some(lambda _: True)
+SYM = some(lambda x: isinstance(x, HySymbol))
+STR = some(lambda x: isinstance(x, HyString))
+
+def sym(wanted):
+    "Parse and skip the given symbol or keyword."
+    if wanted.startswith(":"):
+        return skip(a(HyKeyword(wanted[1:])))
+    return skip(some(lambda x: isinstance(x, HySymbol) and x == wanted))
+
+def whole(parsers):
+    """Parse the parsers in the given list one after another, then
+    expect the end of the input."""
+    if len(parsers) == 0:
+        return finished >> (lambda x: [])
+    if len(parsers) == 1:
+        return parsers[0] + finished >> (lambda x: x[:-1])
+    return reduce(add, parsers) + skip(finished)
+
+def _grouped(group_type, parsers): return (
+    some(lambda x: isinstance(x, group_type)) >>
+    (lambda x: group_type(whole(parsers).parse(x)).replace(x, recursive=False)))
+
+def brackets(*parsers):
+    "Parse the given parsers inside square brackets."
+    return _grouped(HyList, parsers)
+
+def pexpr(*parsers):
+    "Parse the given parsers inside a parenthesized expression."
+    return _grouped(HyExpression, parsers)
+
+def dolike(head):
+    "Parse a `do`-like form."
+    return pexpr(sym(head), many(FORM))
+
+def notpexpr(*disallowed_heads):
+    """Parse any object other than a HyExpression beginning with a
+    HySymbol equal to one of the disallowed_heads."""
+    return some(lambda x: not (
+        isinstance(x, HyExpression) and
+        x and
+        isinstance(x[0], HySymbol) and
+        x[0] in disallowed_heads))
+
+def times(lo, hi, parser):
+    """Parse `parser` several times (`lo` to `hi`) in a row. `hi` can be
+    float('inf'). The result is a list no matter the number of instances."""
+    @Parser
+    def f(tokens, s):
+        result = []
+        for _ in range(lo):
+            (v, s) = parser.run(tokens, s)
+            result.append(v)
+        end = s.max
+        try:
+            for _ in (repeat(1) if isinf(hi) else range(hi - lo)):
+                (v, s) = parser.run(tokens, s)
+                result.append(v)
+        except NoParseError as e:
+            end = e.state.max
+        return result, State(s.pos, end)
+    return f

hy/models.py

@@ -109,13 +109,13 @@ class HyBytes(HyObject, bytes_type):
 _wrappers[bytes_type] = HyBytes
 
 
-class HySymbol(HyString):
+class HySymbol(HyObject, str_type):
     """
-    Hy Symbol. Basically a String.
+    Hy Symbol. Basically a string.
     """
 
-    def __init__(self, string):
+    def __new__(cls, s=None):
-        self += string
+        return super(HySymbol, cls).__new__(cls, s)
 
 _wrappers[bool] = lambda x: HySymbol("True") if x else HySymbol("False")
 _wrappers[type(None)] = lambda foo: HySymbol("None")
@@ -238,10 +238,10 @@ class HySequence(HyObject, list):
     An abstract type for sequence-like models to inherit from.
     """
 
-    def replace(self, other):
+    def replace(self, other, recursive=True):
-        for x in self:
+        if recursive:
-            replace_hy_obj(x, other)
+            for x in self:
-
+                replace_hy_obj(x, other)
         HyObject.replace(self, other)
         return self
 

setup.py

@@ -30,7 +30,7 @@ class Install(install):
                         "." + filename[:-len(".hy")])
         install.run(self)
 
-install_requires = ['rply>=0.7.5', 'astor', 'clint>=0.4']
+install_requires = ['rply>=0.7.5', 'astor', 'funcparserlib>=0.3.6', 'clint>=0.4']
 if os.name == 'nt':
     install_requires.append('pyreadline>=2.1')
 

tests/compilers/test_ast.py

@@ -243,6 +243,7 @@ def test_ast_bad_lambda():
     cant_compile("(fn ())")
     cant_compile("(fn () 1)")
     cant_compile("(fn (x) 1)")
+    cant_compile('(fn "foo")')
 
 
 def test_ast_good_yield():
@@ -442,8 +443,7 @@ def test_lambda_list_keywords_kwonly():
         exception = cant_compile(kwonly_demo)
         assert isinstance(exception, HyTypeError)
         message, = exception.args
-        assert message == ("keyword-only arguments are only "
+        assert message == "&kwonly parameters require Python 3"
-                           "available under Python 3")
 
 
 def test_lambda_list_keywords_mixed():
@@ -451,7 +451,7 @@ def test_lambda_list_keywords_mixed():
     can_compile("(fn [x &rest xs &kwargs kw] (list x xs kw))")
     cant_compile("(fn [x &rest xs &fasfkey {bar \"baz\"}])")
     if PY3:
-        can_compile("(fn [x &rest xs &kwargs kwxs &kwonly kwoxs]"
+        can_compile("(fn [x &rest xs &kwonly kwoxs &kwargs kwxs]"
                     "  (list x xs kwxs kwoxs))")
 
 
@@ -511,7 +511,6 @@ def test_compile_error():
     """Ensure we get compile error in tricky cases"""
     with pytest.raises(HyTypeError) as excinfo:
         can_compile("(fn [] (in [1 2 3]))")
-    assert excinfo.value.message == "`in' needs 2 arguments, got 1"
 
 
 def test_for_compile_error():
@@ -567,6 +566,7 @@ def test_defn():
     cant_compile("(defn \"hy\" [] 1)")
     cant_compile("(defn :hy [] 1)")
     can_compile("(defn &hy [] 1)")
+    cant_compile('(defn hy "foo")')
 
 
 def test_setv_builtins():
@@ -585,11 +585,11 @@ def test_setv_builtins():
 def test_top_level_unquote():
     with pytest.raises(HyTypeError) as excinfo:
         can_compile("(unquote)")
-    assert excinfo.value.message == "`unquote' can't be used at the top-level"
+    assert excinfo.value.message == "The special form 'unquote' is not allowed here"
 
     with pytest.raises(HyTypeError) as excinfo:
         can_compile("(unquote-splice)")
-    assert excinfo.value.message == "`unquote-splice' can't be used at the top-level"
+    assert excinfo.value.message == "The special form 'unquote-splice' is not allowed here"
 
 
 def test_lots_of_comment_lines():

tests/compilers/test_compiler.py

@@ -55,7 +55,7 @@ def test_compiler_yield_return():
                         HyExpression([HySymbol("+"),
                                       HyInteger(1),
                                       HyInteger(1)]))
-    ret = compiler.HyASTCompiler('test').compile_function_def(e)
+    ret = compiler.HyASTCompiler('test').compile_atom(e)
 
     assert len(ret.stmts) == 1
     stmt, = ret.stmts

tests/native_tests/language.hy

@@ -87,8 +87,8 @@
   (assert (= b 2))
   (setv y 0 x 1 y x)
   (assert (= y 1))
-  (try (eval '(setv a 1 b))
+  (with [(pytest.raises HyTypeError)]
-       (except [e [TypeError]] (assert (in "`setv' needs an even number of arguments" (str e))))))
+    (eval '(setv a 1 b))))
 
 
 (defn test-setv-returns-none []
@@ -177,16 +177,6 @@
   None)  ; Avoid https://github.com/hylang/hy/issues/1320
 
 
-(defn test-fn-corner-cases []
-  "NATIVE: tests that fn/defn handles corner cases gracefully"
-  (try (eval '(fn "foo"))
-       (except [e [Exception]] (assert (in "to `fn' must be a list"
-                                          (str e)))))
-  (try (eval '(defn foo "foo"))
-       (except [e [Exception]]
-         (assert (in "takes a parameter list as second" (str e))))))
-
-
 (defn test-for-loop []
   "NATIVE: test for loops"
   (setv count1 0 count2 0)
@@ -280,7 +270,15 @@
         y (range 2)]
     (+ 1 1)
     (else (setv flag (+ flag 2))))
-  (assert (= flag 2)))
+  (assert (= flag 2))
+
+  (setv l [])
+  (defn f []
+    (for [x [4 9 2]]
+      (.append l (* 10 x))
+      (yield x)))
+  (for [_ (f)])
+  (assert (= l [40 90 20])))
 
 
 (defn test-while-loop []
@@ -1388,8 +1386,6 @@
   (import [os.path [basename :as bn]])
   (assert (= bn basename))
 
-  (import [sys])
-
   ;; Multiple stuff to import
   (import sys [os.path [dirname]]
           [os.path :as op]

tests/native_tests/native_macros.hy

@@ -15,42 +15,42 @@
   (rev (.append x 1) (.append x 2) (.append x 3))
   (assert (= x [3 2 1])))
 
-; Macros returning constants
+(defn test-macros-returning-constants []
+  (defmacro an-int [] 42)
+  (assert (= (an-int) 42))
 
-(defmacro an-int [] 42)
+  (defmacro a-true [] True)
-(assert (= (an-int) 42))
+  (assert (= (a-true) True))
+  (defmacro a-false [] False)
+  (assert (= (a-false) False))
 
-(defmacro a-true [] True)
+  (defmacro a-float [] 42.)
-(assert (= (a-true) True))
+  (assert (= (a-float) 42.))
-(defmacro a-false [] False)
-(assert (= (a-false) False))
 
-(defmacro a-float [] 42.)
+  (defmacro a-complex [] 42j)
-(assert (= (a-float) 42.))
+  (assert (= (a-complex) 42j))
 
-(defmacro a-complex [] 42j)
+  (defmacro a-string [] "foo")
-(assert (= (a-complex) 42j))
+  (assert (= (a-string) "foo"))
 
-(defmacro a-string [] "foo")
+  (defmacro a-bytes [] b"foo")
-(assert (= (a-string) "foo"))
+  (assert (= (a-bytes) b"foo"))
 
-(defmacro a-bytes [] b"foo")
+  (defmacro a-list [] [1 2])
-(assert (= (a-bytes) b"foo"))
+  (assert (= (a-list) [1 2]))
 
-(defmacro a-list [] [1 2])
+  (defmacro a-tuple [&rest b] b)
-(assert (= (a-list) [1 2]))
+  (assert (= (a-tuple 1 2) [1 2]))
 
-(defmacro a-tuple [&rest b] b)
+  (defmacro a-dict [] {1 2})
-(assert (= (a-tuple 1 2) [1 2]))
+  (assert (= (a-dict) {1 2}))
 
-(defmacro a-dict [] {1 2})
+  (defmacro a-set [] #{1 2})
-(assert (= (a-dict) {1 2}))
+  (assert (= (a-set) #{1 2}))
 
-(defmacro a-set [] #{1 2})
+  (defmacro a-none [])
-(assert (= (a-set) #{1 2}))
+  (assert (= (a-none) None)))
 
-(defmacro a-none [])
-(assert (= (a-none) None))
 
 ; A macro calling a previously defined function
 (eval-when-compile

tests/resources/pydemo.hy

@@ -19,7 +19,8 @@ Call me Ishmael. Some years ago—never mind how long precisely—having little
 (setv mynan NaN)
 (setv pinf Inf)
 (setv ninf -Inf)
-;(setv mycomplex -Inf+5j) ; https://github.com/berkerpeksag/astor/issues/100
+(setv mycomplex -Inf+5j)
+(setv mycomplex2 NaN-Infj)
 
 (setv num-expr (+ 3 (* 5 2) (- 6 (// 8 2)) (* (+ 1 2) (- 3 5)))) ; = 9
 

tests/test_hy2py.py

@@ -41,6 +41,12 @@ def assert_stuff(m):
     assert m.pinf > 0
     assert math.isinf(m.ninf)
     assert m.ninf < 0
+    assert math.isinf(m.mycomplex.real)
+    assert m.mycomplex.real < 0
+    assert m.mycomplex.imag == 5
+    assert math.isnan(m.mycomplex2.real)
+    assert math.isinf(m.mycomplex2.imag)
+    assert m.mycomplex2.imag < 0
 
     assert m.num_expr == 9