Merge pull request #1314 from Kodiologist/compiler-autobox

Auto-promote values to HyObjects in the compiler

NEWS

@@ -4,6 +4,10 @@ Changes from 0.13.0
    * Single-character "sharp macros" changed to "tag macros", which can have
      longer names
    * Periods are no longer allowed in keywords
+   * `eval` is now a function instead of a special form
+   * The compiler now automatically promotes values to Hy model objects
+     as necessary, so you can write ``(eval `(+ 1 ~n))`` instead of
+     ``(eval `(+ 1 ~(HyInteger n)))``
 
    [ Bug Fixes ]
    * Numeric literals are no longer parsed as symbols when followed by a dot

docs/language/api.rst

@@ -863,27 +863,6 @@ doto
   => collection
   [2 1]
 
-eval
-----
-
-``eval`` evaluates a quoted expression and returns the value. The optional
-second and third arguments specify the dictionary of globals to use and the
-module name. The globals dictionary defaults to ``(local)`` and the module name
-defaults to the name of the current module.
-
-.. code-block:: clj
-
-   => (eval '(print "Hello World"))
-   "Hello World"
-
-If you want to evaluate a string, use ``read-str`` to convert it to a
-form first:
-
-.. code-block:: clj
-
-   => (eval (read-str "(+ 1 1)"))
-   2
-
 
 eval-and-compile
 ----------------

docs/language/core.rst

@@ -230,6 +230,30 @@ Returns ``True`` if *coll* is empty. Equivalent to ``(= 0 (len coll))``.
    False
 
 
+.. _eval-fn:
+
+eval
+----
+
+``eval`` evaluates a quoted expression and returns the value. The optional
+second and third arguments specify the dictionary of globals to use and the
+module name. The globals dictionary defaults to ``(local)`` and the module name
+defaults to the name of the current module.
+
+.. code-block:: clj
+
+   => (eval '(print "Hello World"))
+   "Hello World"
+
+If you want to evaluate a string, use ``read-str`` to convert it to a
+form first:
+
+.. code-block:: clj
+
+   => (eval (read-str "(+ 1 1)"))
+   2
+
+
 .. _every?-fn:
 
 every?

hy/cmdline.py

@@ -15,7 +15,7 @@ import astor.codegen
 
 import hy
 
-from hy.lex import LexException, PrematureEndOfInput, tokenize
+from hy.lex import LexException, PrematureEndOfInput
 from hy.lex.parser import hy_symbol_mangle
 from hy.compiler import HyTypeError
 from hy.importer import (hy_eval, import_buffer_to_module,
@@ -77,12 +77,9 @@ class HyREPL(code.InteractiveConsole):
         global SIMPLE_TRACEBACKS
         try:
             try:
-                tokens = tokenize(source)
+                do = import_buffer_to_hst(source)
             except PrematureEndOfInput:
                 return True
-            do = HyExpression([HySymbol('do')] + tokens)
-            do.start_line = do.end_line = do.start_column = do.end_column = 1
-            do.replace(do)
         except LexException as e:
             if e.source is None:
                 e.source = source

hy/compiler.py

@@ -5,14 +5,15 @@
 
 from hy.models import (HyObject, HyExpression, HyKeyword, HyInteger, HyComplex,
                        HyString, HyBytes, HySymbol, HyFloat, HyList, HySet,
-                       HyDict, HyCons)
+                       HyDict, HyCons, wrap_value)
 from hy.errors import HyCompileError, HyTypeError
 
 from hy.lex.parser import hy_symbol_mangle
 
 import hy.macros
 from hy._compat import (
-    str_type, bytes_type, long_type, PY3, PY34, PY35, raise_empty)
+    str_type, string_types, bytes_type, long_type, PY3, PY34, PY35,
+    raise_empty)
 from hy.macros import require, macroexpand, tag_macroexpand
 import hy.importer
 
@@ -110,6 +111,19 @@ def builds_if(_type, condition):
         return lambda fn: fn
 
 
+def spoof_positions(obj):
+    if not isinstance(obj, HyObject) or isinstance(obj, HyCons):
+        return
+    if not hasattr(obj, "start_column"):
+        obj.start_column = 0
+    if not hasattr(obj, "start_line"):
+        obj.start_line = 0
+    if (hasattr(obj, "__iter__") and
+            not isinstance(obj, (string_types, bytes_type))):
+        for x in obj:
+            spoof_positions(x)
+
+
 class Result(object):
     """
     Smart representation of the result of a hy->AST compilation
@@ -378,23 +392,23 @@ class HyASTCompiler(object):
         ret = Result()
         for module, names in self.imports.items():
             if None in names:
-                ret += self.compile([
+                e = HyExpression([
-                    HyExpression([
                         HySymbol("import"),
                         HySymbol(module),
                     ]).replace(expr)
-                ])
+                spoof_positions(e)
+                ret += self.compile(e)
             names = sorted(name for name in names if name)
             if names:
-                ret += self.compile([
+                e = HyExpression([
-                    HyExpression([
                         HySymbol("import"),
                         HyList([
                             HySymbol(module),
                             HyList([HySymbol(name) for name in names])
                         ])
                     ]).replace(expr)
-                ])
+                spoof_positions(e)
+                ret += self.compile(e)
         self.imports = defaultdict(set)
         return ret.stmts
 
@@ -404,6 +418,11 @@ class HyASTCompiler(object):
             if not isinstance(ret, Result):
                 ret = Result() + ret
             return ret
+        if not isinstance(atom, HyObject):
+            atom = wrap_value(atom)
+            if isinstance(atom, HyObject):
+                spoof_positions(atom)
+                return self.compile_atom(type(atom), atom)
 
     def compile(self, tree):
         try:
@@ -602,12 +621,6 @@ class HyASTCompiler(object):
         ast.copy_location(new_name, name)
         return new_name
 
-    @builds(list)
-    def compile_raw_list(self, entries):
-        ret = self._compile_branch(entries)
-        ret += ret.expr_as_stmt()
-        return ret
-
     def _render_quoted_form(self, form, level):
         """
         Render a quoted form as a new HyExpression.
@@ -706,31 +719,6 @@ class HyASTCompiler(object):
         raise HyTypeError(expr,
                           "`%s' can't be used at the top-level" % expr[0])
 
-    @builds("eval")
-    @checkargs(min=1, max=3)
-    def compile_eval(self, expr):
-        expr.pop(0)
-
-        if not isinstance(expr[0], (HyExpression, HySymbol)):
-            raise HyTypeError(expr, "expression expected as first argument")
-
-        elist = [HySymbol("hy_eval")] + [expr[0]]
-        if len(expr) >= 2:
-            elist.append(expr[1])
-        else:
-            elist.append(HyExpression([HySymbol("locals")]))
-
-        if len(expr) == 3:
-            elist.append(expr[2])
-        else:
-            elist.append(HyString(self.module_name))
-
-        ret = self.compile(HyExpression(elist).replace(expr))
-
-        ret.add_imports("hy.importer", ["hy_eval"])
-
-        return ret
-
     @builds("do")
     def compile_do(self, expression):
         expression.pop(0)
@@ -766,6 +754,7 @@ class HyASTCompiler(object):
         return ret
 
     @builds("try")
+    @checkargs(min=2)
     def compile_try_expression(self, expr):
         expr.pop(0)  # try
 
@@ -1858,6 +1847,7 @@ class HyASTCompiler(object):
         op = ops[expression.pop(0)]
         right_associative = op == ast.Pow
 
+        lineno, col_offset = expression.start_line, expression.start_column
         if right_associative:
             expression = expression[::-1]
         ret = self.compile(expression.pop(0))
@@ -1870,8 +1860,8 @@ class HyASTCompiler(object):
             ret += ast.BinOp(left=left_expr,
                              op=op(),
                              right=right_expr,
-                             lineno=child.start_line,
+                             lineno=lineno,
-                             col_offset=child.start_column)
+                             col_offset=col_offset)
         return ret
 
     @builds("**")
@@ -2619,21 +2609,21 @@ def hy_compile(tree, module_name, root=ast.Module, get_expr=False):
     body = []
     expr = None
 
-    if not (isinstance(tree, HyObject) or type(tree) is list):
+    if not isinstance(tree, HyObject):
-        raise HyCompileError("tree must be a HyObject or a list")
+        tree = wrap_value(tree)
+        if not isinstance(tree, HyObject):
+            raise HyCompileError("`tree` must be a HyObject or capable of "
+                                 "being promoted to one")
+        spoof_positions(tree)
 
-    if isinstance(tree, HyObject) or tree:
+    compiler = HyASTCompiler(module_name)
-        compiler = HyASTCompiler(module_name)
+    result = compiler.compile(tree)
-        result = compiler.compile(tree)
+    expr = result.force_expr
-        expr = result.force_expr
 
-        if not get_expr:
+    if not get_expr:
-            result += result.expr_as_stmt()
+        result += result.expr_as_stmt()
 
-        # We need to test that the type is *exactly* `list` because we don't
+    body = compiler.imports_as_stmts(tree) + result.stmts
-        # want to do `tree[0]` on HyList or such.
-        spoof_tree = tree[0] if type(tree) is list else tree
-        body = compiler.imports_as_stmts(spoof_tree) + result.stmts
 
     ret = root(body=body)
 

hy/core/language.hy

@@ -18,7 +18,8 @@
 (import [hy._compat [long-type]]) ; long for python2, int for python3
 (import [hy.models [HyCons HySymbol HyKeyword]])
 (import [hy.lex [LexException PrematureEndOfInput tokenize]])
-(import [hy.compiler [HyASTCompiler]])
+(import [hy.compiler [HyASTCompiler spoof-positions]])
+(import [hy.importer [hy-eval :as eval]])
 
 (defn butlast [coll]
   "Returns coll except of last element."
@@ -74,8 +75,8 @@
   (import astor)
   (import hy.compiler)
 
-  (fake-source-positions tree)
+  (spoof-positions tree)
-  (setv compiled (hy.compiler.hy_compile tree (calling-module-name)))
+  (setv compiled (hy.compiler.hy-compile tree (calling-module-name)))
   ((if codegen
             astor.codegen.to_source
             astor.dump)
@@ -174,15 +175,6 @@
   "Return true if (pred x) is logical true for every x in coll, else false"
   (all (map pred coll)))
 
-(defn fake-source-positions [tree]
-  "Fake the source positions for a given tree"
-  (if (coll? tree)
-    (for* [subtree tree]
-          (fake-source-positions subtree)))
-  (for* [attr '[start-line end-line start-column end-column]]
-        (if (not (hasattr tree attr))
-          (setattr tree attr 1))))
-
 (defn flatten [coll]
   "Return a single flat list expanding all members of coll"
   (if (coll? coll)
@@ -469,7 +461,7 @@
 (def *exports*
   '[*map accumulate butlast calling-module-name chain coll? combinations
     comp complement compress cons cons? constantly count cycle dec distinct
-    disassemble drop drop-last drop-while empty? even? every? first filter
+    disassemble drop drop-last drop-while empty? eval even? every? first filter
     flatten float? fraction gensym group-by identity inc input instance?
     integer integer? integer-char? interleave interpose islice iterable?
     iterate iterator? juxt keyword keyword? last list* macroexpand

hy/importer.py

@@ -3,7 +3,7 @@
 # license. See the LICENSE.
 
 from hy.compiler import hy_compile, HyTypeError
-from hy.models import HyObject, replace_hy_obj
+from hy.models import HyObject, HyExpression, HySymbol, replace_hy_obj
 from hy.lex import tokenize, LexException
 from hy.errors import HyIOError
 
@@ -14,6 +14,7 @@ import struct
 import imp
 import sys
 import ast
+import inspect
 import os
 import __future__
 
@@ -31,7 +32,7 @@ def ast_compile(ast, filename, mode):
 
 def import_buffer_to_hst(buf):
     """Import content from buf and return a Hy AST."""
-    return tokenize(buf + "\n")
+    return HyExpression([HySymbol("do")] + tokenize(buf + "\n"))
 
 
 def import_file_to_hst(fpath):
@@ -142,7 +143,27 @@ def import_buffer_to_module(module_name, buf):
     return mod
 
 
-def hy_eval(hytree, namespace, module_name, ast_callback=None):
+def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
+    """``eval`` evaluates a quoted expression and returns the value. The optional
+    second and third arguments specify the dictionary of globals to use and the
+    module name. The globals dictionary defaults to ``(local)`` and the module
+    name defaults to the name of the current module.
+
+       => (eval '(print "Hello World"))
+       "Hello World"
+
+    If you want to evaluate a string, use ``read-str`` to convert it to a
+    form first:
+
+       => (eval (read-str "(+ 1 1)"))
+       2"""
+    if namespace is None:
+        frame = inspect.stack()[1][0]
+        namespace = inspect.getargvalues(frame).locals
+    if module_name is None:
+        m = inspect.getmodule(inspect.stack()[1][0])
+        module_name = '__eval__' if m is None else m.__name__
+
     foo = HyObject()
     foo.start_line = 0
     foo.end_line = 0

hy/macros.py

@@ -3,7 +3,7 @@
 # license. See the LICENSE.
 
 from inspect import getargspec, formatargspec
-from hy.models import replace_hy_obj, wrap_value, HyExpression, HyString
+from hy.models import replace_hy_obj, HyExpression, HyString
 
 from hy.errors import HyTypeError, HyMacroExpansionError
 
@@ -196,7 +196,7 @@ def macroexpand_1(tree, compiler):
                     raise HyMacroExpansionError(tree, msg)
 
                 try:
-                    obj = wrap_value(m(*ntree[1:], **opts))
+                    obj = m(*ntree[1:], **opts)
                 except HyTypeError as e:
                     if e.expression is None:
                         e.expression = tree
@@ -225,4 +225,4 @@ def tag_macroexpand(tag, tree, compiler):
             )
 
     expr = tag_macro(tree)
-    return replace_hy_obj(wrap_value(expr), tree)
+    return replace_hy_obj(expr, tree)

hy/models.py

@@ -4,6 +4,7 @@
 
 from __future__ import unicode_literals
 from hy._compat import PY3, str_type, bytes_type, long_type, string_types
+from fractions import Fraction
 
 
 class HyObject(object):
@@ -229,6 +230,8 @@ class HyExpression(HyList):
         return "(%s)" % (" ".join([repr(x) for x in self]))
 
 _wrappers[HyExpression] = lambda e: HyExpression(wrap_value(x) for x in e)
+_wrappers[Fraction] = lambda e: HyExpression(
+    [HySymbol("fraction"), wrap_value(e.numerator), wrap_value(e.denominator)])
 
 
 class HySet(HyList):

tests/compilers/test_ast.py

@@ -7,9 +7,9 @@ from __future__ import unicode_literals
 from hy import HyString
 from hy.models import HyObject
 from hy.compiler import hy_compile
+from hy.importer import import_buffer_to_hst
 from hy.errors import HyCompileError, HyTypeError
 from hy.lex.exceptions import LexException
-from hy.lex import tokenize
 from hy._compat import PY3
 
 import ast
@@ -25,12 +25,12 @@ def _ast_spotcheck(arg, root, secondary):
 
 
 def can_compile(expr):
-    return hy_compile(tokenize(expr), "__main__")
+    return hy_compile(import_buffer_to_hst(expr), "__main__")
 
 
 def cant_compile(expr):
     try:
-        hy_compile(tokenize(expr), "__main__")
+        hy_compile(import_buffer_to_hst(expr), "__main__")
         assert False
     except HyTypeError as e:
         # Anything that can't be compiled should raise a user friendly
@@ -48,8 +48,10 @@ def cant_compile(expr):
 
 def test_ast_bad_type():
     "Make sure AST breakage can happen"
+    class C:
+        pass
     try:
-        hy_compile("foo", "__main__")
+        hy_compile(C(), "__main__")
         assert True is False
     except HyCompileError:
         pass
@@ -252,7 +254,7 @@ def test_ast_require():
 def test_ast_no_pointless_imports():
     def contains_import_from(code):
         return any([isinstance(node, ast.ImportFrom)
-                   for node in hy_compile(tokenize(code), "__main__").body])
+                   for node in can_compile(code).body])
     # `reduce` is a builtin in Python 2, but not Python 3.
     # The version of `map` that returns an iterator is a builtin in
     # Python 3, but not Python 2.
@@ -460,7 +462,7 @@ def test_ast_unicode_strings():
         hy_s.start_line = hy_s.end_line = 0
         hy_s.start_column = hy_s.end_column = 0
 
-        code = hy_compile([hy_s], "__main__")
+        code = hy_compile(hy_s, "__main__")
 
         # code == ast.Module(body=[ast.Expr(value=ast.Str(s=xxx))])
         return code.body[0].value.s
@@ -471,7 +473,7 @@ def test_ast_unicode_strings():
 
 
 def test_ast_unicode_vs_bytes():
-    def f(x): return hy_compile(tokenize(x), "__main__").body[0].value.s
+    def f(x): return can_compile(x).body[0].value.s
     assert f('"hello"') == u"hello"
     assert type(f('"hello"')) is (str if PY3 else unicode)  # noqa
     assert f('b"hello"') == (eval('b"hello"') if PY3 else "hello")

tests/native_tests/language.hy

@@ -5,6 +5,7 @@
 (import [tests.resources [kwtest function-with-a-dash]]
         [os.path [exists isdir isfile]]
         [sys :as systest]
+        re
         [operator [or_]]
         [hy.errors [HyTypeError]]
         pytest)
@@ -1087,10 +1088,42 @@
 (defn test-eval-failure []
   "NATIVE: test eval failure modes"
   ; yo dawg
-  (try (eval '(eval)) (except [e HyTypeError]) (else (assert False)))
+  (try (eval '(eval)) (except [e TypeError]) (else (assert False)))
-  (try (eval '(eval "snafu")) (except [e HyTypeError]) (else (assert False)))
+  (defclass C)
+  (try (eval (C)) (except [e TypeError]) (else (assert False)))
   (try (eval 'False []) (except [e HyTypeError]) (else (assert False)))
-  (try (eval 'False {} 1) (except [e HyTypeError]) (else (assert False))))
+  (try (eval 'False {} 1) (except [e TypeError]) (else (assert False))))
+
+
+(defn test-eval-quasiquote []
+  ; https://github.com/hylang/hy/issues/1174
+
+  (for [x [
+      None False True
+      5 5.1
+      1/2
+      5j 5.1j 2+1j 1.2+3.4j
+      "" b""
+      "apple bloom" b"apple bloom" "⚘" b"\x00"
+      :mykeyword
+      [] #{} {}
+      [1 2 3] #{1 2 3} {"a" 1 "b" 2}]]
+    (assert (= (eval `(identity ~x)) x))
+    (assert (= (eval x) x)))
+
+  ; Tuples wrap to HyLists, not HyExpressions.
+  (assert (= (eval (,)) []))
+  (assert (= (eval (, 1 2 3)) [1 2 3]))
+
+  (assert (= (eval `(+ "a" ~(+ "b" "c"))) "abc"))
+
+  (setv l ["a" "b"])
+  (setv n 1)
+  (assert (= (eval `(get ~l ~n) "b")))
+
+  (setv d {"a" 1 "b" 2})
+  (setv k "b")
+  (assert (= (eval `(get ~d ~k)) 2)))
 
 
 (defn test-import-syntax []
@@ -1366,7 +1399,9 @@
     (assert (= (disassemble '(do (leaky) (leaky) (macros)))
                "Module(\n    body=[\n        Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),\n        Expr(value=Call(func=Name(id='leaky'), args=[], keywords=[], starargs=None, kwargs=None)),\n        Expr(value=Call(func=Name(id='macros'), args=[], keywords=[], starargs=None, kwargs=None))])")))
   (assert (= (disassemble '(do (leaky) (leaky) (macros)) True)
-             "leaky()\nleaky()\nmacros()")))
+             "leaky()\nleaky()\nmacros()"))
+  (assert (= (re.sub r"[L() ]" "" (disassemble `(+ ~(+ 1 1) 40) True))
+             "2+40")))
 
 
 (defn test-attribute-access []

tests/native_tests/native_macros.hy

@@ -84,6 +84,14 @@
   "NATIVE: test macro calling a plain function"
   (assert (= 3 (bar 1 2))))
 
+(defn test-optional-and-apply-in-macro []
+  ; https://github.com/hylang/hy/issues/1154
+  (defn f [&rest args]
+    (+ "f:" (repr args)))
+  (defmacro mac [&optional x]
+   `(apply f [~x]))
+  (assert (= (mac) "f:(None,)")))
+
 (defn test-midtree-yield []
   "NATIVE: test yielding with a returnable"
   (defn kruft [] (yield) (+ 1 1)))