diff --git a/hy/compiler.py b/hy/compiler.py index c718128..394fd13 100644 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -137,13 +137,13 @@ class HyASTCompiler(object): def compile_raw_list(self, entries): return [self.compile(x) for x in entries] - @builds("do") - @builds("progn") + @builds(HySymbol("do")) + @builds(HySymbol("progn")) @checkargs(min=1) def compile_do_expression(self, expr): return [self.compile(x) for x in expr[1:]] - @builds("throw") + @builds(HySymbol("throw")) @checkargs(min=1) def compile_throw_expression(self, expr): expr.pop(0) @@ -156,7 +156,7 @@ class HyASTCompiler(object): inst=None, tback=None) - @builds("try") + @builds(HySymbol("try")) @checkargs(min=1) def compile_try_expression(self, expr): expr.pop(0) # try @@ -175,7 +175,7 @@ class HyASTCompiler(object): finalbody=[], orelse=[]) - @builds("catch") + @builds(HySymbol("catch")) @checkargs(min=2) def compile_catch_expression(self, expr): expr.pop(0) # catch @@ -205,7 +205,7 @@ class HyASTCompiler(object): return self._mangle_branch(branch) return self._mangle_branch([branch]) - @builds("if") + @builds(HySymbol("if")) @checkargs(min=2, max=3) def compile_if_expression(self, expr): expr.pop(0) # if @@ -223,7 +223,7 @@ class HyASTCompiler(object): lineno=expr.start_line, col_offset=expr.start_column) - @builds("print") + @builds(HySymbol("print")) def compile_print_expression(self, expr): call = expr.pop(0) # print if sys.version_info[0] >= 3: @@ -243,7 +243,7 @@ class HyASTCompiler(object): values=[self.compile(x) for x in expr], nl=True) - @builds("assert") + @builds(HySymbol("assert")) @checkargs(1) def compile_assert_expression(self, expr): expr.pop(0) # assert @@ -253,7 +253,7 @@ class HyASTCompiler(object): lineno=e.start_line, col_offset=e.start_column) - @builds("lambda") + @builds(HySymbol("lambda")) @checkargs(min=2) def compile_lambda_expression(self, expr): expr.pop(0) @@ -276,12 +276,12 @@ class HyASTCompiler(object): kw_defaults=[]), body=self.compile(body)) - @builds("pass") + @builds(HySymbol("pass")) @checkargs(0) def compile_pass_expression(self, expr): return ast.Pass(lineno=expr.start_line, col_offset=expr.start_column) - @builds("yield") + @builds(HySymbol("yield")) @checkargs(1) def compile_yield_expression(self, expr): expr.pop(0) @@ -290,7 +290,7 @@ class HyASTCompiler(object): lineno=expr.start_line, col_offset=expr.start_column) - @builds("import") + @builds(HySymbol("import")) def compile_import_expression(self, expr): expr.pop(0) # index return ast.Import( @@ -298,7 +298,7 @@ class HyASTCompiler(object): col_offset=expr.start_column, names=[ast.alias(name=str(x), asname=None) for x in expr]) - @builds("import_as") + @builds(HySymbol("import-as")) def compile_import_as_expression(self, expr): expr.pop(0) # index modlist = [expr[i:i + 2] for i in range(0, len(expr), 2)] @@ -309,7 +309,7 @@ class HyASTCompiler(object): names=[ast.alias(name=str(x[0]), asname=str(x[1])) for x in modlist]) - @builds("import_from") + @builds(HySymbol("import-from")) @checkargs(min=1) def compile_import_from_expression(self, expr): expr.pop(0) # index @@ -320,7 +320,7 @@ class HyASTCompiler(object): names=[ast.alias(name=str(x), asname=None) for x in expr], level=0) - @builds("get") + @builds(HySymbol("get")) @checkargs(2) def compile_index_expression(self, expr): expr.pop(0) # index @@ -334,7 +334,7 @@ class HyASTCompiler(object): slice=ast.Index(value=sli), ctx=ast.Load()) - @builds("slice") + @builds(HySymbol("slice")) @checkargs(min=1, max=3) def compile_slice_expression(self, expr): expr.pop(0) # index @@ -357,7 +357,7 @@ class HyASTCompiler(object): step=None), ctx=ast.Load()) - @builds("assoc") + @builds(HySymbol("assoc")) @checkargs(3) def compile_assoc_expression(self, expr): expr.pop(0) # assoc @@ -378,7 +378,7 @@ class HyASTCompiler(object): ctx=ast.Store())], value=self.compile(val)) - @builds("decorate_with") + @builds(HySymbol("decorate-with")) @checkargs(min=1) def compile_decorate_expression(self, expr): expr.pop(0) # decorate-with @@ -388,7 +388,7 @@ class HyASTCompiler(object): fn.decorator_list = [self.compile(x) for x in expr] return fn - @builds("with_as") + @builds(HySymbol("with-as")) @checkargs(min=2) def compile_with_as_expression(self, expr): expr.pop(0) # with-as @@ -407,7 +407,7 @@ class HyASTCompiler(object): return ret - @builds(",") + @builds(HySymbol(",")) def compile_tuple(self, expr): expr.pop(0) return ast.Tuple(elts=[self.compile(x) for x in expr], @@ -415,7 +415,7 @@ class HyASTCompiler(object): col_offset=expr.start_column, ctx=ast.Load()) - @builds("list_comp") + @builds(HySymbol("list-comp")) @checkargs(min=2, max=3) def compile_list_comprehension(self, expr): # (list-comp expr (target iter) cond?) @@ -450,7 +450,7 @@ class HyASTCompiler(object): name.ctx = ast.Store() return name - @builds("kwapply") + @builds(HySymbol("kwapply")) @checkargs(2) def compile_kwapply_expression(self, expr): expr.pop(0) # kwapply @@ -465,8 +465,8 @@ class HyASTCompiler(object): return call - @builds("not") - @builds("~") + @builds(HySymbol("not")) + @builds(HySymbol("~")) @checkargs(1) def compile_unary_operator(self, expression): ops = {"not": ast.Not, @@ -478,23 +478,23 @@ class HyASTCompiler(object): lineno=operator.start_line, col_offset=operator.start_column) - @builds("=") - @builds("!=") - @builds("<") - @builds("<=") - @builds(">") - @builds(">=") - @builds("is") - @builds("in") - @builds("is_not") - @builds("not_in") + @builds(HySymbol("=")) + @builds(HySymbol("!=")) + @builds(HySymbol("<")) + @builds(HySymbol("<=")) + @builds(HySymbol(">")) + @builds(HySymbol(">=")) + @builds(HySymbol("is")) + @builds(HySymbol("in")) + @builds(HySymbol("is-not")) + @builds(HySymbol("not-in")) @checkargs(min=2) def compile_compare_op_expression(self, expression): ops = {"=": ast.Eq, "!=": ast.NotEq, "<": ast.Lt, "<=": ast.LtE, ">": ast.Gt, ">=": ast.GtE, - "is": ast.Is, "is_not": ast.IsNot, - "in": ast.In, "not_in": ast.NotIn} + "is": ast.Is, "is-not": ast.IsNot, + "in": ast.In, "not-in": ast.NotIn} inv = expression.pop(0) op = ops[inv] @@ -507,11 +507,11 @@ class HyASTCompiler(object): lineno=e.start_line, col_offset=e.start_column) - @builds("+") - @builds("%") - @builds("-") - @builds("/") - @builds("*") + @builds(HySymbol("+")) + @builds(HySymbol("%")) + @builds(HySymbol("-")) + @builds(HySymbol("/")) + @builds(HySymbol("*")) @checkargs(min=2) def compile_maths_expression(self, expression): # operator = Mod | Pow | LShift | RShift | BitOr | @@ -578,9 +578,9 @@ class HyASTCompiler(object): lineno=expression.start_line, col_offset=expression.start_column) - @builds("def") - @builds("setf") - @builds("setv") + @builds(HySymbol("def")) + @builds(HySymbol("setf")) + @builds(HySymbol("setv")) @checkargs(2) def compile_def_expression(self, expression): expression.pop(0) # "def" @@ -602,7 +602,7 @@ class HyASTCompiler(object): col_offset=expression.start_column, targets=[name], value=what) - @builds("foreach") + @builds(HySymbol("foreach")) @checkargs(min=1) def compile_for_expression(self, expression): ret_status = self.returnable @@ -626,7 +626,7 @@ class HyASTCompiler(object): self.returnable = ret_status return ret - @builds("while") + @builds(HySymbol("while")) @checkargs(min=2) def compile_while_expression(self, expr): expr.pop(0) # "while" @@ -647,7 +647,7 @@ class HyASTCompiler(object): lineno=expr.start_line, col_offset=expr.start_column) - @builds("fn") + @builds(HySymbol("fn")) @checkargs(min=2) def compile_fn_expression(self, expression): expression.pop(0) # fn diff --git a/hy/contrib/meth.py b/hy/contrib/meth.py index 4f64863..3dc59a8 100644 --- a/hy/contrib/meth.py +++ b/hy/contrib/meth.py @@ -40,7 +40,7 @@ def router(tree, rkwargs=None): route = HyExpression([HySymbol("kwapply"), route, HyDict({HyString("methods"): rkwargs})]) - return HyExpression([HySymbol("decorate_with"), route, tree]) + return HyExpression([HySymbol("decorate-with"), route, tree]) @macro("route") diff --git a/hy/core/bootstrap.py b/hy/core/bootstrap.py index e77fb02..7442149 100644 --- a/hy/core/bootstrap.py +++ b/hy/core/bootstrap.py @@ -80,7 +80,7 @@ def for_macro(tree): return root -@macro("_>") +@macro("->") def threading_macro(tree): tree.pop(0) ret = tree.pop(0) diff --git a/hy/core/mangles.py b/hy/core/mangles.py index 69744ec..c82728c 100644 --- a/hy/core/mangles.py +++ b/hy/core/mangles.py @@ -41,7 +41,7 @@ class HoistableMangle(hy.mangle.Mangle): class FunctionMangle(HoistableMangle): hoistable = ["fn"] - ignore = ["def", "decorate_with", "setf", "setv"] + ignore = ["def", "decorate-with", "setf", "setv"] def __init__(self): self.series = 0 diff --git a/hy/importer.py b/hy/importer.py index 21b6d74..11aa653 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -24,6 +24,7 @@ from hy.core import process from py_compile import wr_long, MAGIC +import io import marshal import imp import sys @@ -43,7 +44,7 @@ def import_buffer_to_hst(fd): def import_file_to_hst(fpath): - return import_buffer_to_hst(open(fpath, 'r')) + return import_buffer_to_hst(io.open(fpath, 'rU', encoding='utf-8')) def import_file_to_ast(fpath): diff --git a/hy/lex/states.py b/hy/lex/states.py index 72caba6..677cb3b 100644 --- a/hy/lex/states.py +++ b/hy/lex/states.py @@ -58,16 +58,7 @@ def _resolve_atom(obj): "null": "None", } - if obj in table: - return HySymbol(table[obj]) - - if obj.startswith("*") and obj.endswith("*") and obj != "*": - obj = obj[1:-1].upper() - - if "-" in obj and obj != "-": - obj = obj.replace("-", "_") - - return HySymbol(obj) + return HySymbol(table.get(obj, obj)) class State(object): diff --git a/hy/macros.py b/hy/macros.py index 3eda0c7..8eb0f7c 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -22,13 +22,14 @@ from hy.models.expression import HyExpression from hy.models.string import HyString from hy.models.dict import HyDict from hy.models.list import HyList +from hy.models.symbol import HySymbol _hy_macros = {} def macro(name): def _(fn): - _hy_macros[name] = fn + _hy_macros[HySymbol(name)] = fn return fn return _ diff --git a/hy/models/symbol.py b/hy/models/symbol.py index 7a1685c..a6ccb87 100644 --- a/hy/models/symbol.py +++ b/hy/models/symbol.py @@ -1,4 +1,5 @@ # Copyright (c) 2013 Paul Tagliamonte +# Copyright (c) 2013 Julien Danjou # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -18,6 +19,8 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import base64 + from hy.models.string import HyString @@ -25,6 +28,13 @@ class HySymbol(HyString): """ Hy Symbol. Basically a String. """ + def encode_symbol(self): + return ("_hy_symbol_" + + base64.b64encode( + self.encode( + 'unicode-escape')).decode('ascii')) - def __init__(self, string): - self += string + def __str__(self): + if all([(c.isalnum() or c == "_" or c == ".") for c in self]): + return super(HyString, self).__str__() + return self.encode_symbol() diff --git a/tests/native_tests/language.hy b/tests/native_tests/language.hy index 2f539ec..27069b6 100644 --- a/tests/native_tests/language.hy +++ b/tests/native_tests/language.hy @@ -160,6 +160,21 @@ (assert (= (.join " " ["one" "two"]) "one two"))) +(defn test-symbol-utf-8 [] + "NATIVE: test symbol encoded" + (let [[♥ "love"] + [⚘ "flower"]] + (assert (= (+ ⚘ ♥) "flowerlove")))) + + +(defn test-symbol-dash [] + "NATIVE: test symbol encoded" + (let [[♥-♥ "doublelove"] + [-_- "what?"]] + (assert (= ♥-♥ "doublelove")) + (assert (= -_- "what?")))) + + (defn test-exceptions [] "NATIVE: test Exceptions" (try