Yaltik/hy
2
0
Fork 0
Code Pull Requests Releases Activity

Using backslashes for now; punycode soon.

Browse Source
This commit is contained in:
Paul R. Tagliamonte 2013-04-06 21:33:52 -04:00
parent bac3f6991c
commit 5a96089266
4 changed files with 47 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hy/compiler.py
View File

@ -40,6 +40,13 @@ class HyCompileError(HyError):
_compile_table = {} _compile_table = {}
def ast_str(foobar):
if sys.version_info[0] >= 3:
return str(foobar)
return str(foobar.encode("ascii", 'backslashreplace'))
def builds(_type): def builds(_type):
def _dec(fn): def _dec(fn):
_compile_table[_type] = fn _compile_table[_type] = fn
@ -188,7 +195,7 @@ class HyASTCompiler(object):
# #
# We'll just make sure it's a pure "string", and let it work # We'll just make sure it's a pure "string", and let it work
# it's magic. # it's magic.
name = str(name) name = ast_str(name)
else: else:
# Python2 requires an ast.Name, set to ctx Store. # Python2 requires an ast.Name, set to ctx Store.
name = self._storeize(self.compile(name)) name = self._storeize(self.compile(name))
@ -264,7 +271,7 @@ class HyASTCompiler(object):
lineno=expr.start_line, lineno=expr.start_line,
col_offset=expr.start_column, col_offset=expr.start_column,
args=ast.arguments(args=[ args=ast.arguments(args=[
ast.Name(arg=str(x), id=str(x), ast.Name(arg=ast_str(x), id=ast_str(x),
ctx=ast.Param(), ctx=ast.Param(),
lineno=x.start_line, lineno=x.start_line,
col_offset=x.start_column) col_offset=x.start_column)
@ -296,7 +303,7 @@ class HyASTCompiler(object):
return ast.Import( return ast.Import(
lineno=expr.start_line, lineno=expr.start_line,
col_offset=expr.start_column, col_offset=expr.start_column,
names=[ast.alias(name=str(x), asname=None) for x in expr]) names=[ast.alias(name=ast_str(x), asname=None) for x in expr])
@builds("import_as") @builds("import_as")
def compile_import_as_expression(self, expr): def compile_import_as_expression(self, expr):
@ -305,9 +312,9 @@ class HyASTCompiler(object):
return ast.Import( return ast.Import(
lineno=expr.start_line, lineno=expr.start_line,
col_offset=expr.start_column, col_offset=expr.start_column,
module=str(expr.pop(0)), module=ast_str(expr.pop(0)),
names=[ast.alias(name=str(x[0]), names=[ast.alias(name=ast_str(x[0]),
asname=str(x[1])) for x in modlist]) asname=ast_str(x[1])) for x in modlist])
@builds("import_from") @builds("import_from")
@checkargs(min=1) @checkargs(min=1)
@ -316,8 +323,8 @@ class HyASTCompiler(object):
return ast.ImportFrom( return ast.ImportFrom(
lineno=expr.start_line, lineno=expr.start_line,
col_offset=expr.start_column, col_offset=expr.start_column,
module=str(expr.pop(0)), module=ast_str(expr.pop(0)),
names=[ast.alias(name=str(x), asname=None) for x in expr], names=[ast.alias(name=ast_str(x), asname=None) for x in expr],
level=0) level=0)
@builds("get") @builds("get")
@ -460,7 +467,7 @@ class HyASTCompiler(object):
if type(call) != ast.Call: if type(call) != ast.Call:
raise TypeError("kwapplying a non-call") raise TypeError("kwapplying a non-call")
call.keywords = [ast.keyword(arg=str(x), call.keywords = [ast.keyword(arg=ast_str(x),
value=self.compile(kwargs[x])) for x in kwargs] value=self.compile(kwargs[x])) for x in kwargs]
return call return call
@ -551,7 +558,7 @@ class HyASTCompiler(object):
lineno=expr.start_line, lineno=expr.start_line,
col_offset=expr.start_column, col_offset=expr.start_column,
value=self.compile(obj), value=self.compile(obj),
attr=str(fn), attr=ast_str(fn),
ctx=ast.Load()), ctx=ast.Load()),
args=[self.compile(x) for x in expr], args=[self.compile(x) for x in expr],
keywords=[], keywords=[],
@ -592,7 +599,7 @@ class HyASTCompiler(object):
# We special case a FunctionDef, since we can define by setting # We special case a FunctionDef, since we can define by setting
# FunctionDef's .name attribute, rather then foo == anon_fn. This # FunctionDef's .name attribute, rather then foo == anon_fn. This
# helps keep things clean. # helps keep things clean.
what.name = str(name) what.name = ast_str(name)
return what return what
name = self._storeize(self.compile(name)) name = self._storeize(self.compile(name))
@ -666,7 +673,7 @@ class HyASTCompiler(object):
args=ast.arguments( args=ast.arguments(
args=[ args=[
ast.Name( ast.Name(
arg=str(x), id=str(x), arg=ast_str(x), id=ast_str(x),
ctx=ast.Param(), ctx=ast.Param(),
lineno=x.start_line, lineno=x.start_line,
col_offset=x.start_column) col_offset=x.start_column)
@ -700,19 +707,19 @@ class HyASTCompiler(object):
lineno=symbol.start_line, lineno=symbol.start_line,
col_offset=symbol.start_column, col_offset=symbol.start_column,
value=self.compile_symbol(glob), value=self.compile_symbol(glob),
attr=str(local), attr=ast_str(local),
ctx=ast.Load() ctx=ast.Load()
) )
return ast.Name(id=str(symbol), return ast.Name(id=ast_str(symbol),
arg=str(symbol), arg=ast_str(symbol),
ctx=ast.Load(), ctx=ast.Load(),
lineno=symbol.start_line, lineno=symbol.start_line,
col_offset=symbol.start_column) col_offset=symbol.start_column)
@builds(HyString) @builds(HyString)
def compile_string(self, string): def compile_string(self, string):
return ast.Str(s=str(string), lineno=string.start_line, return ast.Str(s=ast_str(string), lineno=string.start_line,
col_offset=string.start_column) col_offset=string.start_column)
@builds(HyDict) @builds(HyDict)

15
hy/importer.py
View File

@ -19,11 +19,12 @@
# DEALINGS IN THE SOFTWARE. # DEALINGS IN THE SOFTWARE.
from hy.compiler import hy_compile from hy.compiler import hy_compile
from hy.lex import tokenize
from hy.core import process
from py_compile import wr_long, MAGIC from py_compile import wr_long, MAGIC
from hy.core import process
from hy.lex import tokenize
from io import open
import marshal import marshal
import imp import imp
import sys import sys
@ -43,18 +44,12 @@ def import_buffer_to_hst(fd):
def import_file_to_hst(fpath): def import_file_to_hst(fpath):
return import_buffer_to_hst(open(fpath, 'r')) return import_buffer_to_hst(open(fpath, 'r', encoding='utf-8'))
def import_file_to_ast(fpath): def import_file_to_ast(fpath):
tree = import_file_to_hst(fpath) tree = import_file_to_hst(fpath)
try: ast = hy_compile(tree)
ast = hy_compile(tree)
except Exception as e:
print("Compilation error at %s:%d,%d"
% (fpath, e.start_line, e.start_column))
print("Compilation error: " + e.message)
raise e.exception
return ast return ast

8
hy/models/string.py
View File

@ -23,12 +23,12 @@ import sys
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
_str_type = str str_type = str
else: else:
_str_type = unicode str_type = unicode
class HyString(HyObject, _str_type): class HyString(HyObject, str_type):
""" """
Generic Hy String object. Helpful to store string literals from Hy Generic Hy String object. Helpful to store string literals from Hy
scripts. It's either a ``str`` or a ``unicode``, depending on the scripts. It's either a ``str`` or a ``unicode``, depending on the
@ -36,5 +36,5 @@ class HyString(HyObject, _str_type):
""" """
def __new__(cls, value): def __new__(cls, value):
obj = _str_type.__new__(cls, value) obj = str_type.__new__(cls, value)
return obj return obj

15
tests/native_tests/language.hy
View File

@ -331,6 +331,21 @@
(assert (= y 123))) (assert (= y 123)))
(defn test-symbol-utf-8 []
"NATIVE: test symbol encoded"
(let [[♥ "love"]
[⚘ "flower"]]
(assert (= (+ ⚘ ♥) "flowerlove"))))
(defn test-symbol-dash []
"NATIVE: test symbol encoded"
(let [[♥-♥ "doublelove"]
[-_- "what?"]]
(assert (= ♥-♥ "doublelove"))
(assert (= -_- "what?"))))
; FEATURE: native hy-eval ; FEATURE: native hy-eval
; ;
; - related to bug #64 ; - related to bug #64