Implement new importer using PEP-302 semantics
Python 3.x is patched in a way that integrates `.hy` source files into Pythons default `importlib` machinery. In Python 2.7, a PEP-302 "importer" and "loader" is implemented according to the standard `import` logic (via `pkgutil` and later pure-Python `imp` package code). In both cases, the entry-point for the loaders is through `sys.path_hooks` only. As well, the import semantics have been updated all throughout to utilize `importlib` and follow aspects of PEP-420. This, along with some light patches, should allow for basic use of `runpy`, `py_compile` and `reload`. In all cases, if a `.hy` file is shadowed by a `.py`, Hy will silently use `.hy`.
This commit is contained in:
Brandon T. Willard
parent
c92fb3c494
commit
87a5b117a1
47
conftest.py
47
conftest.py
@ -1,15 +1,50 @@
|
||||
import os
|
||||
import importlib
|
||||
|
||||
import py
|
||||
import pytest
|
||||
import hy
|
||||
import os
|
||||
from hy._compat import PY3, PY35, PY36
|
||||
|
||||
NATIVE_TESTS = os.path.join("", "tests", "native_tests", "")
|
||||
|
||||
_fspath_pyimport = py.path.local.pyimport
|
||||
|
||||
|
||||
def pytest_ignore_collect(path, config):
|
||||
return (("py3_only" in path.basename and not PY3)
|
||||
or ("py35_only" in path.basename and not PY35)
|
||||
or ("py36_only" in path.basename and not PY36))
|
||||
|
||||
|
||||
def pyimport_patch_mismatch(self, **kwargs):
|
||||
"""Lame fix for https://github.com/pytest-dev/py/issues/195"""
|
||||
try:
|
||||
return _fspath_pyimport(self, **kwargs)
|
||||
except py.path.local.ImportMismatchError:
|
||||
pkgpath = self.pypkgpath()
|
||||
if pkgpath is None:
|
||||
pkgroot = self.dirpath()
|
||||
modname = self.purebasename
|
||||
else:
|
||||
pkgroot = pkgpath.dirpath()
|
||||
names = self.new(ext="").relto(pkgroot).split(self.sep)
|
||||
if names[-1] == "__init__":
|
||||
names.pop()
|
||||
modname = ".".join(names)
|
||||
|
||||
res = importlib.import_module(modname)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
py.path.local.pyimport = pyimport_patch_mismatch
|
||||
|
||||
|
||||
def pytest_collect_file(parent, path):
|
||||
if (path.ext == ".hy"
|
||||
and NATIVE_TESTS in path.dirname + os.sep
|
||||
and path.basename != "__init__.hy"
|
||||
and not ("py3_only" in path.basename and not PY3)
|
||||
and not ("py35_only" in path.basename and not PY35)
|
||||
and not ("py36_only" in path.basename and not PY36)):
|
||||
return pytest.Module(path, parent)
|
||||
and path.basename != "__init__.hy"):
|
||||
|
||||
pytest_mod = pytest.Module(path, parent)
|
||||
return pytest_mod
|
||||
|
||||
@ -231,7 +231,7 @@ But assignments via ``import`` are always hoisted to normal Python scope, and
|
||||
likewise, ``defclass`` will assign the class to the Python scope,
|
||||
even if it shares the name of a let binding.
|
||||
|
||||
Use ``__import__`` and ``type`` (or whatever metaclass) instead,
|
||||
Use ``importlib.import_module`` and ``type`` (or whatever metaclass) instead,
|
||||
if you must avoid this hoisting.
|
||||
|
||||
The ``let`` macro takes two parameters: a list defining *variables*
|
||||
|
||||
@ -6,18 +6,6 @@ try:
|
||||
import __builtin__ as builtins
|
||||
except ImportError:
|
||||
import builtins # NOQA
|
||||
try:
|
||||
from py_compile import MAGIC, wr_long
|
||||
except ImportError:
|
||||
# py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4
|
||||
from importlib.util import MAGIC_NUMBER as MAGIC # NOQA
|
||||
|
||||
def wr_long(f, x):
|
||||
"""Internal; write a 32-bit int to a file in little-endian order."""
|
||||
f.write(bytes([x & 0xff,
|
||||
(x >> 8) & 0xff,
|
||||
(x >> 16) & 0xff,
|
||||
(x >> 24) & 0xff]))
|
||||
import sys, keyword
|
||||
|
||||
PY3 = sys.version_info[0] >= 3
|
||||
@ -60,3 +48,8 @@ def isidentifier(x):
|
||||
except T.TokenError:
|
||||
return False
|
||||
return len(tokens) == 2 and tokens[0][0] == T.NAME
|
||||
|
||||
try:
|
||||
FileNotFoundError = FileNotFoundError
|
||||
except NameError:
|
||||
FileNotFoundError = IOError
|
||||
|
||||
113
hy/cmdline.py
113
hy/cmdline.py
@ -9,26 +9,21 @@ import code
|
||||
import ast
|
||||
import sys
|
||||
import os
|
||||
import io
|
||||
import importlib
|
||||
import py_compile
|
||||
import runpy
|
||||
|
||||
import astor.code_gen
|
||||
|
||||
import hy
|
||||
|
||||
from hy.lex import LexException, PrematureEndOfInput, mangle
|
||||
from hy.compiler import HyTypeError
|
||||
from hy.importer import (hy_eval, import_buffer_to_module,
|
||||
import_file_to_ast, import_file_to_hst,
|
||||
import_buffer_to_ast, import_buffer_to_hst)
|
||||
from hy.completer import completion
|
||||
from hy.completer import Completer
|
||||
|
||||
from hy.errors import HyIOError
|
||||
|
||||
from hy.compiler import HyTypeError, hy_compile
|
||||
from hy.importer import hy_eval, hy_parse
|
||||
from hy.completer import completion, Completer
|
||||
from hy.macros import macro, require
|
||||
from hy.models import HyExpression, HyString, HySymbol
|
||||
|
||||
from hy._compat import builtins, PY3
|
||||
from hy._compat import builtins, PY3, FileNotFoundError
|
||||
|
||||
|
||||
class HyQuitter(object):
|
||||
@ -47,6 +42,7 @@ class HyQuitter(object):
|
||||
pass
|
||||
raise SystemExit(code)
|
||||
|
||||
|
||||
builtins.quit = HyQuitter('quit')
|
||||
builtins.exit = HyQuitter('exit')
|
||||
|
||||
@ -88,7 +84,7 @@ class HyREPL(code.InteractiveConsole):
|
||||
|
||||
try:
|
||||
try:
|
||||
do = import_buffer_to_hst(source)
|
||||
do = hy_parse(source)
|
||||
except PrematureEndOfInput:
|
||||
return True
|
||||
except LexException as e:
|
||||
@ -202,25 +198,8 @@ def pretty_error(func, *args, **kw):
|
||||
|
||||
|
||||
def run_command(source):
|
||||
pretty_error(import_buffer_to_module, "__main__", source)
|
||||
return 0
|
||||
|
||||
|
||||
def run_module(mod_name):
|
||||
from hy.importer import MetaImporter
|
||||
pth = MetaImporter().find_on_path(mod_name)
|
||||
if pth is not None:
|
||||
sys.argv = [pth] + sys.argv
|
||||
return run_file(pth)
|
||||
|
||||
print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def run_file(filename):
|
||||
from hy.importer import import_file_to_module
|
||||
pretty_error(import_file_to_module, "__main__", filename)
|
||||
tree = hy_parse(source)
|
||||
pretty_error(hy_eval, tree, module_name="__main__")
|
||||
return 0
|
||||
|
||||
|
||||
@ -252,7 +231,7 @@ def run_repl(hr=None, **kwargs):
|
||||
def run_icommand(source, **kwargs):
|
||||
hr = HyREPL(**kwargs)
|
||||
if os.path.exists(source):
|
||||
with open(source, "r") as f:
|
||||
with io.open(source, "r", encoding='utf-8') as f:
|
||||
source = f.read()
|
||||
filename = source
|
||||
else:
|
||||
@ -320,7 +299,7 @@ def cmdline_handler(scriptname, argv):
|
||||
SIMPLE_TRACEBACKS = False
|
||||
|
||||
# reset sys.argv like Python
|
||||
sys.argv = options.args + module_args or [""]
|
||||
# sys.argv = [sys.argv[0]] + options.args + module_args
|
||||
|
||||
if options.E:
|
||||
# User did "hy -E ..."
|
||||
@ -332,7 +311,9 @@ def cmdline_handler(scriptname, argv):
|
||||
|
||||
if options.mod:
|
||||
# User did "hy -m ..."
|
||||
return run_module(options.mod)
|
||||
sys.argv = [sys.argv[0]] + options.args + module_args
|
||||
runpy.run_module(options.mod, run_name='__main__', alter_sys=True)
|
||||
return 0
|
||||
|
||||
if options.icommand:
|
||||
# User did "hy -i ..."
|
||||
@ -347,9 +328,11 @@ def cmdline_handler(scriptname, argv):
|
||||
else:
|
||||
# User did "hy <filename>"
|
||||
try:
|
||||
return run_file(options.args[0])
|
||||
except HyIOError as e:
|
||||
print("hy: Can't open file '{0}': [Errno {1}] {2}\n".format(
|
||||
sys.argv = options.args
|
||||
runpy.run_path(options.args[0], run_name='__main__')
|
||||
return 0
|
||||
except FileNotFoundError as e:
|
||||
print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
|
||||
e.filename, e.errno, e.strerror), file=sys.stderr)
|
||||
sys.exit(e.errno)
|
||||
|
||||
@ -359,27 +342,45 @@ def cmdline_handler(scriptname, argv):
|
||||
|
||||
# entry point for cmd line script "hy"
|
||||
def hy_main():
|
||||
sys.path.insert(0, "")
|
||||
sys.exit(cmdline_handler("hy", sys.argv))
|
||||
|
||||
|
||||
# entry point for cmd line script "hyc"
|
||||
def hyc_main():
|
||||
from hy.importer import write_hy_as_pyc
|
||||
parser = argparse.ArgumentParser(prog="hyc")
|
||||
parser.add_argument("files", metavar="FILE", nargs='+',
|
||||
help="file to compile")
|
||||
parser.add_argument("files", metavar="FILE", nargs='*',
|
||||
help=('File(s) to compile (use STDIN if only'
|
||||
' "-" or nothing is provided)'))
|
||||
parser.add_argument("-v", action="version", version=VERSION)
|
||||
|
||||
options = parser.parse_args(sys.argv[1:])
|
||||
|
||||
for file in options.files:
|
||||
rv = 0
|
||||
if len(options.files) == 0 or (
|
||||
len(options.files) == 1 and options.files[0] == '-'):
|
||||
while True:
|
||||
filename = sys.stdin.readline()
|
||||
if not filename:
|
||||
break
|
||||
filename = filename.rstrip('\n')
|
||||
try:
|
||||
print("Compiling %s" % file)
|
||||
pretty_error(write_hy_as_pyc, file)
|
||||
except IOError as x:
|
||||
print("hyc: Can't open file '{0}': [Errno {1}] {2}\n".format(
|
||||
x.filename, x.errno, x.strerror), file=sys.stderr)
|
||||
sys.exit(x.errno)
|
||||
py_compile.compile(filename, doraise=True)
|
||||
except py_compile.PyCompileError as error:
|
||||
rv = 1
|
||||
sys.stderr.write("%s\n" % error.msg)
|
||||
except OSError as error:
|
||||
rv = 1
|
||||
sys.stderr.write("%s\n" % error)
|
||||
else:
|
||||
for filename in options.files:
|
||||
try:
|
||||
print("Compiling %s" % filename)
|
||||
py_compile.compile(filename, doraise=True)
|
||||
except py_compile.PyCompileError as error:
|
||||
# return value to indicate at least one failure
|
||||
rv = 1
|
||||
sys.stderr.write("%s\n" % error.msg)
|
||||
return rv
|
||||
|
||||
|
||||
# entry point for cmd line script "hy2py"
|
||||
@ -403,14 +404,14 @@ def hy2py_main():
|
||||
|
||||
options = parser.parse_args(sys.argv[1:])
|
||||
|
||||
stdin_text = None
|
||||
if options.FILE is None or options.FILE == '-':
|
||||
stdin_text = sys.stdin.read()
|
||||
source = sys.stdin.read()
|
||||
else:
|
||||
with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
|
||||
source = source_file.read()
|
||||
|
||||
hst = pretty_error(hy_parse, source)
|
||||
if options.with_source:
|
||||
hst = (pretty_error(import_file_to_hst, options.FILE)
|
||||
if stdin_text is None
|
||||
else pretty_error(import_buffer_to_hst, stdin_text))
|
||||
# need special printing on Windows in case the
|
||||
# codepage doesn't support utf-8 characters
|
||||
if PY3 and platform.system() == "Windows":
|
||||
@ -424,9 +425,7 @@ def hy2py_main():
|
||||
print()
|
||||
print()
|
||||
|
||||
_ast = (pretty_error(import_file_to_ast, options.FILE, module_name)
|
||||
if stdin_text is None
|
||||
else pretty_error(import_buffer_to_ast, stdin_text, module_name))
|
||||
_ast = pretty_error(hy_compile, hst, module_name)
|
||||
if options.with_ast:
|
||||
if PY3 and platform.system() == "Windows":
|
||||
_print_for_windows(astor.dump_tree(_ast))
|
||||
|
||||
@ -1151,7 +1151,7 @@ class HyASTCompiler(object):
|
||||
ret += node(
|
||||
expr, module=module or None, names=names, level=level)
|
||||
else: # root == "require"
|
||||
__import__(module)
|
||||
importlib.import_module(module)
|
||||
require(module, self.module_name,
|
||||
assignments=assignments, prefix=prefix)
|
||||
|
||||
|
||||
@ -299,7 +299,7 @@ But assignments via `import` are always hoisted to normal Python scope, and
|
||||
likewise, `defclass` will assign the class to the Python scope,
|
||||
even if it shares the name of a let binding.
|
||||
|
||||
Use __import__ and type (or whatever metaclass) instead,
|
||||
Use `import_module` and `type` (or whatever metaclass) instead,
|
||||
if you must avoid this hoisting.
|
||||
|
||||
Function arguments can shadow let bindings in their body,
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
;;; These macros form the hy language
|
||||
;;; They are automatically required in every module, except inside hy.core
|
||||
|
||||
(import [importlib [import-module]])
|
||||
|
||||
(import [hy.models [HyList HySymbol]])
|
||||
|
||||
@ -247,13 +248,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
|
||||
Use ``#doc foo`` instead for help with tag macro ``#foo``.
|
||||
Use ``(help foo)`` instead for help with runtime objects."
|
||||
`(try
|
||||
(help (. (__import__ "hy")
|
||||
(help (. (import-module "hy")
|
||||
macros
|
||||
_hy_macros
|
||||
[__name__]
|
||||
['~symbol]))
|
||||
(except [KeyError]
|
||||
(help (. (__import__ "hy")
|
||||
(help (. (import-module "hy")
|
||||
macros
|
||||
_hy_macros
|
||||
[None]
|
||||
@ -264,13 +265,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol."
|
||||
|
||||
Gets help for a tag macro function available in this module."
|
||||
`(try
|
||||
(help (. (__import__ "hy")
|
||||
(help (. (import-module "hy")
|
||||
macros
|
||||
_hy_tag
|
||||
[__name__]
|
||||
['~symbol]))
|
||||
(except [KeyError]
|
||||
(help (. (__import__ "hy")
|
||||
(help (. (import-module "hy")
|
||||
macros
|
||||
_hy_tag
|
||||
[None]
|
||||
|
||||
705
hy/importer.py
705
hy/importer.py
@ -4,158 +4,75 @@
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from hy.compiler import hy_compile, HyTypeError
|
||||
from hy.models import HyExpression, HySymbol
|
||||
from hy.lex import tokenize, LexException
|
||||
from hy.errors import HyIOError
|
||||
|
||||
from io import open
|
||||
import re
|
||||
import marshal
|
||||
import struct
|
||||
import imp
|
||||
import sys
|
||||
import os
|
||||
import ast
|
||||
import inspect
|
||||
import os
|
||||
import pkgutil
|
||||
import re
|
||||
import io
|
||||
import runpy
|
||||
import types
|
||||
import tempfile
|
||||
import importlib
|
||||
import __future__
|
||||
|
||||
from hy._compat import PY3, PY37, MAGIC, builtins, long_type, wr_long
|
||||
from hy._compat import string_types
|
||||
from hy.errors import HyTypeError
|
||||
from hy.compiler import hy_compile
|
||||
from hy.lex import tokenize, LexException
|
||||
from hy.models import HyExpression, HySymbol
|
||||
from hy._compat import string_types, PY3
|
||||
|
||||
|
||||
hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION |
|
||||
__future__.CO_FUTURE_PRINT_FUNCTION)
|
||||
|
||||
|
||||
def ast_compile(ast, filename, mode):
|
||||
"""Compile AST.
|
||||
Like Python's compile, but with some special flags."""
|
||||
flags = (__future__.CO_FUTURE_DIVISION |
|
||||
__future__.CO_FUTURE_PRINT_FUNCTION)
|
||||
return compile(ast, filename, mode, flags)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ast : instance of `ast.AST`
|
||||
|
||||
filename : str
|
||||
Filename used for run-time error messages
|
||||
|
||||
mode: str
|
||||
`compile` mode parameter
|
||||
|
||||
Returns
|
||||
-------
|
||||
out : instance of `types.CodeType`
|
||||
"""
|
||||
return compile(ast, filename, mode, hy_ast_compile_flags)
|
||||
|
||||
|
||||
def import_buffer_to_hst(buf):
|
||||
"""Import content from buf and return a Hy AST."""
|
||||
return HyExpression([HySymbol("do")] + tokenize(buf + "\n"))
|
||||
def hy_parse(source):
|
||||
"""Parse a Hy source string.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source: string
|
||||
Source code to parse.
|
||||
|
||||
def import_file_to_hst(fpath):
|
||||
"""Import content from fpath and return a Hy AST."""
|
||||
try:
|
||||
with open(fpath, 'r', encoding='utf-8') as f:
|
||||
buf = f.read()
|
||||
# Strip the shebang line, if there is one.
|
||||
buf = re.sub(r'\A#!.*', '', buf)
|
||||
return import_buffer_to_hst(buf)
|
||||
except IOError as e:
|
||||
raise HyIOError(e.errno, e.strerror, e.filename)
|
||||
|
||||
|
||||
def import_buffer_to_ast(buf, module_name):
|
||||
""" Import content from buf and return a Python AST."""
|
||||
return hy_compile(import_buffer_to_hst(buf), module_name)
|
||||
|
||||
|
||||
def import_file_to_ast(fpath, module_name):
|
||||
"""Import content from fpath and return a Python AST."""
|
||||
return hy_compile(import_file_to_hst(fpath), module_name)
|
||||
|
||||
|
||||
def import_file_to_module(module_name, fpath, loader=None):
|
||||
"""Import Hy source from fpath and put it into a Python module.
|
||||
|
||||
If there's an up-to-date byte-compiled version of this module, load that
|
||||
instead. Otherwise, byte-compile the module once we're done loading it, if
|
||||
we can.
|
||||
|
||||
Return the module."""
|
||||
|
||||
module = None
|
||||
|
||||
bytecode_path = get_bytecode_path(fpath)
|
||||
try:
|
||||
source_mtime = int(os.stat(fpath).st_mtime)
|
||||
with open(bytecode_path, 'rb') as bc_f:
|
||||
# The first 4 bytes are the magic number for the version of Python
|
||||
# that compiled this bytecode.
|
||||
bytecode_magic = bc_f.read(4)
|
||||
# Python 3.7 introduced a new flags entry in the header structure.
|
||||
if PY37:
|
||||
bc_f.read(4)
|
||||
# The next 4 bytes, interpreted as a little-endian 32-bit integer,
|
||||
# are the mtime of the corresponding source file.
|
||||
bytecode_mtime, = struct.unpack('<i', bc_f.read(4))
|
||||
except (IOError, OSError):
|
||||
pass
|
||||
else:
|
||||
if bytecode_magic == MAGIC and bytecode_mtime >= source_mtime:
|
||||
# It's a cache hit. Load the byte-compiled version.
|
||||
if PY3:
|
||||
# As of Python 3.6, imp.load_compiled still exists, but it's
|
||||
# deprecated. So let's use SourcelessFileLoader instead.
|
||||
from importlib.machinery import SourcelessFileLoader
|
||||
module = (SourcelessFileLoader(module_name, bytecode_path).
|
||||
load_module(module_name))
|
||||
else:
|
||||
module = imp.load_compiled(module_name, bytecode_path)
|
||||
|
||||
if not module:
|
||||
# It's a cache miss, so load from source.
|
||||
sys.modules[module_name] = None
|
||||
try:
|
||||
_ast = import_file_to_ast(fpath, module_name)
|
||||
module = imp.new_module(module_name)
|
||||
module.__file__ = os.path.normpath(fpath)
|
||||
code = ast_compile(_ast, fpath, "exec")
|
||||
if not os.environ.get('PYTHONDONTWRITEBYTECODE'):
|
||||
try:
|
||||
write_code_as_pyc(fpath, code)
|
||||
except (IOError, OSError):
|
||||
# We failed to save the bytecode, probably because of a
|
||||
# permissions issue. The user only asked to import the
|
||||
# file, so don't bug them about it.
|
||||
pass
|
||||
eval(code, module.__dict__)
|
||||
except (HyTypeError, LexException) as e:
|
||||
if e.source is None:
|
||||
with open(fpath, 'rt') as fp:
|
||||
e.source = fp.read()
|
||||
e.filename = fpath
|
||||
raise
|
||||
except Exception:
|
||||
sys.modules.pop(module_name, None)
|
||||
raise
|
||||
sys.modules[module_name] = module
|
||||
module.__name__ = module_name
|
||||
|
||||
module.__file__ = os.path.normpath(fpath)
|
||||
if loader:
|
||||
module.__loader__ = loader
|
||||
if is_package(module_name):
|
||||
module.__path__ = []
|
||||
module.__package__ = module_name
|
||||
else:
|
||||
module.__package__ = module_name.rpartition('.')[0]
|
||||
|
||||
return module
|
||||
|
||||
|
||||
def import_buffer_to_module(module_name, buf):
|
||||
try:
|
||||
_ast = import_buffer_to_ast(buf, module_name)
|
||||
mod = imp.new_module(module_name)
|
||||
eval(ast_compile(_ast, "", "exec"), mod.__dict__)
|
||||
except (HyTypeError, LexException) as e:
|
||||
if e.source is None:
|
||||
e.source = buf
|
||||
e.filename = '<stdin>'
|
||||
raise
|
||||
return mod
|
||||
Returns
|
||||
-------
|
||||
out : instance of `types.CodeType`
|
||||
"""
|
||||
source = re.sub(r'\A#!.*', '', source)
|
||||
return HyExpression([HySymbol("do")] + tokenize(source + "\n"))
|
||||
|
||||
|
||||
def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
|
||||
"""``eval`` evaluates a quoted expression and returns the value. The optional
|
||||
second and third arguments specify the dictionary of globals to use and the
|
||||
module name. The globals dictionary defaults to ``(local)`` and the module
|
||||
name defaults to the name of the current module.
|
||||
"""Evaluates a quoted expression and returns the value.
|
||||
|
||||
The optional second and third arguments specify the dictionary of globals
|
||||
to use and the module name. The globals dictionary defaults to ``(local)``
|
||||
and the module name defaults to the name of the current module.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
=> (eval '(print "Hello World"))
|
||||
"Hello World"
|
||||
@ -164,7 +81,31 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
|
||||
form first:
|
||||
|
||||
=> (eval (read-str "(+ 1 1)"))
|
||||
2"""
|
||||
2
|
||||
|
||||
Parameters
|
||||
----------
|
||||
hytree: a Hy expression tree
|
||||
Source code to parse.
|
||||
|
||||
namespace: dict, optional
|
||||
Namespace in which to evaluate the Hy tree. Defaults to the calling
|
||||
frame.
|
||||
|
||||
module_name: str, optional
|
||||
Name of the module to which the Hy tree is assigned. Defaults to
|
||||
the calling frame's module, if any, and '__eval__' otherwise.
|
||||
|
||||
ast_callback: callable, optional
|
||||
A callback that is passed the Hy compiled tree and resulting
|
||||
expression object, in that order, after compilation but before
|
||||
evaluation.
|
||||
|
||||
Returns
|
||||
-------
|
||||
out : Result of evaluating the Hy compiled tree.
|
||||
|
||||
"""
|
||||
if namespace is None:
|
||||
frame = inspect.stack()[1][0]
|
||||
namespace = inspect.getargvalues(frame).locals
|
||||
@ -199,89 +140,427 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None):
|
||||
return eval(ast_compile(expr, "<eval>", "eval"), namespace)
|
||||
|
||||
|
||||
def write_hy_as_pyc(fname):
|
||||
_ast = import_file_to_ast(fname,
|
||||
os.path.basename(os.path.splitext(fname)[0]))
|
||||
code = ast_compile(_ast, fname, "exec")
|
||||
write_code_as_pyc(fname, code)
|
||||
def cache_from_source(source_path):
|
||||
"""Get the cached bytecode file name for a given source file name.
|
||||
|
||||
This function's name is set to mirror Python 3.x's
|
||||
`importlib.util.cache_from_source`, which is also used when available.
|
||||
|
||||
def write_code_as_pyc(fname, code):
|
||||
st = os.stat(fname)
|
||||
timestamp = long_type(st.st_mtime)
|
||||
Parameters
|
||||
----------
|
||||
source_path : str
|
||||
Path of the source file
|
||||
|
||||
cfile = get_bytecode_path(fname)
|
||||
try:
|
||||
os.makedirs(os.path.dirname(cfile))
|
||||
except (IOError, OSError):
|
||||
pass
|
||||
|
||||
with builtins.open(cfile, 'wb') as fc:
|
||||
fc.write(MAGIC)
|
||||
if PY37:
|
||||
# With PEP 552, the header structure has a new flags field
|
||||
# that we need to fill in. All zeros preserve the legacy
|
||||
# behaviour, but should we implement reproducible builds,
|
||||
# this is where we'd add the information.
|
||||
wr_long(fc, 0)
|
||||
wr_long(fc, timestamp)
|
||||
Returns
|
||||
-------
|
||||
out : str
|
||||
Path of the corresponding bytecode file that may--or may
|
||||
not--actually exist.
|
||||
"""
|
||||
if PY3:
|
||||
wr_long(fc, st.st_size)
|
||||
marshal.dump(code, fc)
|
||||
|
||||
|
||||
class MetaLoader(object):
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
def load_module(self, fullname):
|
||||
if fullname in sys.modules:
|
||||
return sys.modules[fullname]
|
||||
|
||||
if not self.path:
|
||||
return
|
||||
|
||||
return import_file_to_module(fullname, self.path, self)
|
||||
|
||||
|
||||
class MetaImporter(object):
|
||||
def find_on_path(self, fullname):
|
||||
fls = ["%s/__init__.hy", "%s.hy"]
|
||||
dirpath = "/".join(fullname.split("."))
|
||||
|
||||
for pth in sys.path:
|
||||
pth = os.path.abspath(pth)
|
||||
for fp in fls:
|
||||
composed_path = fp % ("%s/%s" % (pth, dirpath))
|
||||
if os.path.exists(composed_path):
|
||||
return composed_path
|
||||
|
||||
def find_module(self, fullname, path=None):
|
||||
path = self.find_on_path(fullname)
|
||||
if path:
|
||||
return MetaLoader(path)
|
||||
|
||||
|
||||
sys.meta_path.insert(0, MetaImporter())
|
||||
sys.path.insert(0, "")
|
||||
|
||||
|
||||
def is_package(module_name):
|
||||
mpath = os.path.join(*module_name.split("."))
|
||||
for path in map(os.path.abspath, sys.path):
|
||||
if os.path.exists(os.path.join(path, mpath, "__init__.hy")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_bytecode_path(source_path):
|
||||
if PY3:
|
||||
import importlib.util
|
||||
return importlib.util.cache_from_source(source_path)
|
||||
elif hasattr(imp, "cache_from_source"):
|
||||
return imp.cache_from_source(source_path)
|
||||
else:
|
||||
# If source_path has a file extension, replace it with ".pyc".
|
||||
# Otherwise, just append ".pyc".
|
||||
d, f = os.path.split(source_path)
|
||||
return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
|
||||
|
||||
|
||||
def _get_code_from_file(run_name, fname=None):
|
||||
"""A patch of `runpy._get_code_from_file` that will also compile Hy
|
||||
code.
|
||||
|
||||
This version will read and cache bytecode for Hy files. It operates
|
||||
normally otherwise.
|
||||
"""
|
||||
if fname is None and run_name is not None:
|
||||
fname = run_name
|
||||
|
||||
if fname.endswith('.hy'):
|
||||
full_fname = os.path.abspath(fname)
|
||||
fname_path, fname_file = os.path.split(full_fname)
|
||||
modname = os.path.splitext(fname_file)[0]
|
||||
sys.path.insert(0, fname_path)
|
||||
try:
|
||||
loader = pkgutil.get_loader(modname)
|
||||
code = loader.get_code(modname)
|
||||
finally:
|
||||
sys.path.pop(0)
|
||||
else:
|
||||
with open(fname, "rb") as f:
|
||||
code = pkgutil.read_code(f)
|
||||
if code is None:
|
||||
with open(fname, "rb") as f:
|
||||
source = f.read().decode('utf-8')
|
||||
code = compile(source, fname, 'exec')
|
||||
|
||||
return (code, fname) if PY3 else code
|
||||
|
||||
|
||||
_runpy_get_code_from_file = runpy._get_code_from_file
|
||||
runpy._get_code_from_file = _get_code_from_file
|
||||
|
||||
if PY3:
|
||||
importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy')
|
||||
_py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code
|
||||
|
||||
def _hy_source_to_code(self, data, path, _optimize=-1):
|
||||
if os.path.isfile(path) and path.endswith('.hy'):
|
||||
source = data.decode("utf-8")
|
||||
try:
|
||||
hy_tree = hy_parse(source)
|
||||
data = hy_compile(hy_tree, self.name)
|
||||
except (HyTypeError, LexException) as e:
|
||||
if e.source is None:
|
||||
e.source = source
|
||||
e.filename = path
|
||||
raise
|
||||
|
||||
return _py_source_to_code(self, data, path, _optimize=_optimize)
|
||||
|
||||
importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code
|
||||
|
||||
# This is actually needed; otherwise, pre-created finders assigned to the
|
||||
# current dir (i.e. `''`) in `sys.path` will not catch absolute imports of
|
||||
# directory-local modules!
|
||||
sys.path_importer_cache.clear()
|
||||
|
||||
# Do this one just in case?
|
||||
importlib.invalidate_caches()
|
||||
|
||||
# XXX: These and the 2.7 counterparts below aren't truly cross-compliant.
|
||||
# They're useful for testing, though.
|
||||
HyImporter = importlib.machinery.FileFinder
|
||||
HyLoader = importlib.machinery.SourceFileLoader
|
||||
|
||||
else:
|
||||
import imp
|
||||
import py_compile
|
||||
import marshal
|
||||
import struct
|
||||
import traceback
|
||||
|
||||
from pkgutil import ImpImporter, ImpLoader
|
||||
|
||||
# 100x better!
|
||||
HY_SOURCE = imp.PY_SOURCE * 100
|
||||
|
||||
class HyLoader(ImpLoader, object):
|
||||
def __init__(self, fullname, filename, fileobj=None, etc=None):
|
||||
"""This constructor is designed for some compatibility with
|
||||
SourceFileLoader."""
|
||||
if etc is None and filename is not None:
|
||||
if filename.endswith('.hy'):
|
||||
etc = ('.hy', 'U', HY_SOURCE)
|
||||
if fileobj is None:
|
||||
fileobj = open(filename, 'rU')
|
||||
|
||||
super(HyLoader, self).__init__(fullname, fileobj, filename, etc)
|
||||
|
||||
def exec_module(self, module, fullname=None):
|
||||
fullname = self._fix_name(fullname)
|
||||
ast = self.get_code(fullname)
|
||||
eval(ast, module.__dict__)
|
||||
|
||||
def load_module(self, fullname=None):
|
||||
"""Same as `pkgutil.ImpLoader`, with an extra check for Hy
|
||||
source"""
|
||||
fullname = self._fix_name(fullname)
|
||||
mod_type = self.etc[2]
|
||||
mod = None
|
||||
pkg_path = os.path.join(self.filename, '__init__.hy')
|
||||
if mod_type == HY_SOURCE or (
|
||||
mod_type == imp.PKG_DIRECTORY and
|
||||
os.path.isfile(pkg_path)):
|
||||
|
||||
if fullname in sys.modules:
|
||||
mod = sys.modules[fullname]
|
||||
else:
|
||||
mod = sys.modules.setdefault(
|
||||
fullname, imp.new_module(fullname))
|
||||
|
||||
# TODO: Should we set these only when not in `sys.modules`?
|
||||
if mod_type == imp.PKG_DIRECTORY:
|
||||
mod.__file__ = pkg_path
|
||||
mod.__path__ = [self.filename]
|
||||
mod.__package__ = fullname
|
||||
else:
|
||||
# mod.__path__ = self.filename
|
||||
mod.__file__ = self.get_filename(fullname)
|
||||
mod.__package__ = '.'.join(fullname.split('.')[:-1])
|
||||
|
||||
# TODO: Set `mod.__doc__`.
|
||||
mod.__name__ = fullname
|
||||
|
||||
self.exec_module(mod, fullname=fullname)
|
||||
|
||||
if mod is None:
|
||||
self._reopen()
|
||||
try:
|
||||
mod = imp.load_module(fullname, self.file, self.filename,
|
||||
self.etc)
|
||||
finally:
|
||||
if self.file:
|
||||
self.file.close()
|
||||
|
||||
mod.__loader__ = self
|
||||
return mod
|
||||
|
||||
def _reopen(self):
|
||||
"""Same as `pkgutil.ImpLoader`, with an extra check for Hy
|
||||
source"""
|
||||
super(HyLoader, self)._reopen()
|
||||
|
||||
# Add the Hy case...
|
||||
if self.file and self.file.closed:
|
||||
mod_type = self.etc[2]
|
||||
if mod_type == HY_SOURCE:
|
||||
self.file = io.open(self.filename, 'rU', encoding='utf-8')
|
||||
|
||||
def byte_compile_hy(self, fullname=None):
|
||||
fullname = self._fix_name(fullname)
|
||||
if fullname is None:
|
||||
fullname = self.fullname
|
||||
try:
|
||||
hy_source = self.get_source(fullname)
|
||||
hy_tree = hy_parse(hy_source)
|
||||
ast = hy_compile(hy_tree, fullname)
|
||||
code = compile(ast, self.filename, 'exec',
|
||||
hy_ast_compile_flags)
|
||||
except (HyTypeError, LexException) as e:
|
||||
if e.source is None:
|
||||
e.source = hy_source
|
||||
e.filename = self.filename
|
||||
raise
|
||||
|
||||
if not sys.dont_write_bytecode:
|
||||
try:
|
||||
hyc_compile(code)
|
||||
except IOError:
|
||||
pass
|
||||
return code
|
||||
|
||||
def get_code(self, fullname=None):
|
||||
"""Same as `pkgutil.ImpLoader`, with an extra check for Hy
|
||||
source"""
|
||||
fullname = self._fix_name(fullname)
|
||||
mod_type = self.etc[2]
|
||||
if mod_type == HY_SOURCE:
|
||||
# Looks like we have to manually check for--and update--
|
||||
# the bytecode.
|
||||
t_py = long(os.stat(self.filename).st_mtime)
|
||||
pyc_file = cache_from_source(self.filename)
|
||||
if os.path.isfile(pyc_file):
|
||||
t_pyc = long(os.stat(pyc_file).st_mtime)
|
||||
|
||||
if t_pyc is not None and t_pyc >= t_py:
|
||||
with open(pyc_file, 'rb') as f:
|
||||
if f.read(4) == imp.get_magic():
|
||||
t = struct.unpack('<I', f.read(4))[0]
|
||||
if t == t_py:
|
||||
self.code = marshal.load(f)
|
||||
|
||||
if self.code is None:
|
||||
# There's no existing bytecode, or bytecode timestamp
|
||||
# is older than the source file's.
|
||||
self.code = self.byte_compile_hy(fullname)
|
||||
|
||||
if self.code is None:
|
||||
super(HyLoader, self).get_code(fullname=fullname)
|
||||
|
||||
return self.code
|
||||
|
||||
def get_source(self, fullname=None):
|
||||
"""Same as `pkgutil.ImpLoader`, with an extra check for Hy
|
||||
source"""
|
||||
fullname = self._fix_name(fullname)
|
||||
mod_type = self.etc[2]
|
||||
if self.source is None and mod_type == HY_SOURCE:
|
||||
self._reopen()
|
||||
try:
|
||||
self.source = self.file.read()
|
||||
finally:
|
||||
self.file.close()
|
||||
|
||||
if self.source is None:
|
||||
super(HyLoader, self).get_source(fullname=fullname)
|
||||
|
||||
return self.source
|
||||
|
||||
def get_filename(self, fullname=None):
|
||||
"""Same as `pkgutil.ImpLoader`, with an extra check for Hy
|
||||
source"""
|
||||
fullname = self._fix_name(fullname)
|
||||
if self.etc[2] == HY_SOURCE:
|
||||
return self.filename
|
||||
|
||||
if self.filename is None:
|
||||
filename = super(HyLoader, self).get_filename(fullname=fullname)
|
||||
|
||||
return filename
|
||||
|
||||
def _get_delegate(self):
|
||||
return HyImporter(self.filename).find_module('__init__')
|
||||
|
||||
class HyImporter(ImpImporter, object):
|
||||
|
||||
def __init__(self, path=None):
|
||||
# We need to be strict about the types of files this importer will
|
||||
# handle. To start, if the path is not the current directory in
|
||||
# (represented by '' in `sys.path`), then it must be a supported
|
||||
# file type or a directory. If it isn't, this importer is not
|
||||
# suitable: throw an exception.
|
||||
|
||||
if path == '' or os.path.isdir(path) or (
|
||||
os.path.isfile(path) and path.endswith('.hy')):
|
||||
self.path = path
|
||||
else:
|
||||
raise ImportError('Invalid path: {}'.format(path))
|
||||
|
||||
def find_loader(self, fullname):
|
||||
return self.find_module(fullname, path=None)
|
||||
|
||||
def find_module(self, fullname, path=None):
|
||||
|
||||
subname = fullname.split(".")[-1]
|
||||
|
||||
if subname != fullname and self.path is None:
|
||||
return None
|
||||
|
||||
if self.path is None:
|
||||
path = None
|
||||
else:
|
||||
path = [os.path.realpath(self.path)]
|
||||
|
||||
fileobj, file_path, etc = None, None, None
|
||||
|
||||
# The following are excerpts from the later pure Python
|
||||
# implementations of the `imp` module (e.g. in Python 3.6).
|
||||
if path is None:
|
||||
path = sys.path
|
||||
|
||||
for entry in path:
|
||||
if (os.path.isfile(entry) and subname == '__main__' and
|
||||
entry.endswith('.hy')):
|
||||
file_path = entry
|
||||
fileobj = io.open(file_path, 'rU', encoding='utf-8')
|
||||
etc = ('.hy', 'U', HY_SOURCE)
|
||||
break
|
||||
else:
|
||||
file_path = os.path.join(entry, subname)
|
||||
path_init = os.path.join(file_path, '__init__.hy')
|
||||
if os.path.isfile(path_init):
|
||||
fileobj = None
|
||||
etc = ('', '', imp.PKG_DIRECTORY)
|
||||
break
|
||||
|
||||
file_path = file_path + '.hy'
|
||||
if os.path.isfile(file_path):
|
||||
fileobj = io.open(file_path, 'rU', encoding='utf-8')
|
||||
etc = ('.hy', 'U', HY_SOURCE)
|
||||
break
|
||||
else:
|
||||
try:
|
||||
fileobj, file_path, etc = imp.find_module(subname, path)
|
||||
except (ImportError, IOError):
|
||||
return None
|
||||
|
||||
|
||||
return HyLoader(fullname, file_path, fileobj, etc)
|
||||
|
||||
sys.path_hooks.append(HyImporter)
|
||||
sys.path_importer_cache.clear()
|
||||
|
||||
_py_compile_compile = py_compile.compile
|
||||
|
||||
def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False):
|
||||
"""Write a Hy file, or code object, to pyc.
|
||||
|
||||
This is a patched version of Python 2.7's `py_compile.compile`.
|
||||
|
||||
Also, it tries its best to write the bytecode file atomically.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file_or_code : str or instance of `types.CodeType`
|
||||
A filename for a Hy or Python source file or its corresponding code
|
||||
object.
|
||||
cfile : str, optional
|
||||
The filename to use for the bytecode file. If `None`, use the
|
||||
standard bytecode filename determined by `cache_from_source`.
|
||||
dfile : str, optional
|
||||
The filename to use for compile-time errors.
|
||||
doraise : bool, default False
|
||||
If `True` raise compilation exceptions; otherwise, ignore them.
|
||||
|
||||
Returns
|
||||
-------
|
||||
out : str
|
||||
The resulting bytecode file name. Python 3.x returns this, but
|
||||
Python 2.7 doesn't; this function does for convenience.
|
||||
"""
|
||||
|
||||
if isinstance(file_or_code, types.CodeType):
|
||||
codeobject = file_or_code
|
||||
filename = codeobject.co_filename
|
||||
else:
|
||||
filename = file_or_code
|
||||
|
||||
with open(filename, 'rb') as f:
|
||||
source_str = f.read().decode('utf-8')
|
||||
|
||||
try:
|
||||
flags = None
|
||||
if filename.endswith('.hy'):
|
||||
hy_tree = hy_parse(source_str)
|
||||
source = hy_compile(hy_tree, '<hyc_compile>')
|
||||
flags = hy_ast_compile_flags
|
||||
|
||||
codeobject = compile(source, dfile or filename, 'exec', flags)
|
||||
except Exception as err:
|
||||
if isinstance(err, (HyTypeError, LexException)) and err.source is None:
|
||||
err.source = source_str
|
||||
err.filename = filename
|
||||
|
||||
py_exc = py_compile.PyCompileError(err.__class__, err,
|
||||
dfile or filename)
|
||||
if doraise:
|
||||
raise py_exc
|
||||
else:
|
||||
traceback.print_exc()
|
||||
return
|
||||
|
||||
timestamp = long(os.stat(filename).st_mtime)
|
||||
|
||||
if cfile is None:
|
||||
cfile = cache_from_source(filename)
|
||||
|
||||
f = tempfile.NamedTemporaryFile('wb', dir=os.path.split(cfile)[0],
|
||||
delete=False)
|
||||
try:
|
||||
f.write('\0\0\0\0')
|
||||
f.write(struct.pack('<I', timestamp))
|
||||
f.write(marshal.dumps(codeobject))
|
||||
f.flush()
|
||||
f.seek(0, 0)
|
||||
f.write(imp.get_magic())
|
||||
|
||||
# Make sure it's written to disk.
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
f.close()
|
||||
|
||||
# Rename won't replace an existing dest on Windows.
|
||||
if os.name == 'nt' and os.path.isfile(cfile):
|
||||
os.unlink(cfile)
|
||||
|
||||
os.rename(f.name, cfile)
|
||||
except OSError:
|
||||
try:
|
||||
os.unlink(f.name)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return cfile
|
||||
|
||||
py_compile.compile = hyc_compile
|
||||
|
||||
20
hy/macros.py
20
hy/macros.py
@ -1,6 +1,10 @@
|
||||
# Copyright 2018 the authors.
|
||||
# This file is part of Hy, which is free software licensed under the Expat
|
||||
# license. See the LICENSE.
|
||||
import pkgutil
|
||||
import importlib
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
from hy._compat import PY3
|
||||
import hy.inspect
|
||||
@ -10,8 +14,6 @@ from hy._compat import str_type
|
||||
|
||||
from hy.errors import HyTypeError, HyMacroExpansionError
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
CORE_MACROS = [
|
||||
"hy.core.bootstrap",
|
||||
]
|
||||
@ -120,20 +122,14 @@ def load_macros(module_name):
|
||||
Other modules get the macros from CORE_MACROS and EXTRA_MACROS.
|
||||
|
||||
"""
|
||||
|
||||
def _import(module, module_name=module_name):
|
||||
"__import__ a module, avoiding recursions"
|
||||
if module != module_name:
|
||||
__import__(module)
|
||||
|
||||
for module in CORE_MACROS:
|
||||
_import(module)
|
||||
importlib.import_module(module)
|
||||
|
||||
if module_name.startswith("hy.core"):
|
||||
return
|
||||
|
||||
for module in EXTRA_MACROS:
|
||||
_import(module)
|
||||
importlib.import_module(module)
|
||||
|
||||
|
||||
def make_empty_fn_copy(fn):
|
||||
@ -159,8 +155,8 @@ def make_empty_fn_copy(fn):
|
||||
def macroexpand(tree, compiler, once=False):
|
||||
"""Expand the toplevel macros for the `tree`.
|
||||
|
||||
Load the macros from the given `module_name`, then expand the (top-level)
|
||||
macros in `tree` until we no longer can.
|
||||
Load the macros from the given `compiler.module_name`, then expand the
|
||||
(top-level) macros in `tree` until we no longer can.
|
||||
|
||||
"""
|
||||
load_macros(compiler.module_name)
|
||||
|
||||
@ -7,8 +7,7 @@ from __future__ import unicode_literals
|
||||
|
||||
from hy import HyString
|
||||
from hy.models import HyObject
|
||||
from hy.compiler import hy_compile
|
||||
from hy.importer import hy_eval, import_buffer_to_hst
|
||||
from hy.importer import hy_compile, hy_eval, hy_parse
|
||||
from hy.errors import HyCompileError, HyTypeError
|
||||
from hy.lex.exceptions import LexException
|
||||
from hy._compat import PY3
|
||||
@ -16,6 +15,7 @@ from hy._compat import PY3
|
||||
import ast
|
||||
import pytest
|
||||
|
||||
|
||||
def _ast_spotcheck(arg, root, secondary):
|
||||
if "." in arg:
|
||||
local, full = arg.split(".", 1)
|
||||
@ -26,16 +26,16 @@ def _ast_spotcheck(arg, root, secondary):
|
||||
|
||||
|
||||
def can_compile(expr):
|
||||
return hy_compile(import_buffer_to_hst(expr), "__main__")
|
||||
return hy_compile(hy_parse(expr), "__main__")
|
||||
|
||||
|
||||
def can_eval(expr):
|
||||
return hy_eval(import_buffer_to_hst(expr))
|
||||
return hy_eval(hy_parse(expr))
|
||||
|
||||
|
||||
def cant_compile(expr):
|
||||
try:
|
||||
hy_compile(import_buffer_to_hst(expr), "__main__")
|
||||
hy_compile(hy_parse(expr), "__main__")
|
||||
assert False
|
||||
except HyTypeError as e:
|
||||
# Anything that can't be compiled should raise a user friendly
|
||||
|
||||
@ -2,35 +2,43 @@
|
||||
# This file is part of Hy, which is free software licensed under the Expat
|
||||
# license. See the LICENSE.
|
||||
|
||||
import hy
|
||||
from hy.importer import (import_file_to_module, import_buffer_to_ast,
|
||||
MetaLoader, get_bytecode_path)
|
||||
from hy.errors import HyTypeError
|
||||
import os
|
||||
import ast
|
||||
import tempfile
|
||||
import importlib
|
||||
import runpy
|
||||
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
|
||||
import hy
|
||||
from hy.errors import HyTypeError
|
||||
from hy.compiler import hy_compile
|
||||
from hy.importer import hy_parse, HyLoader, cache_from_source
|
||||
|
||||
|
||||
def test_basics():
|
||||
"Make sure the basics of the importer work"
|
||||
import_file_to_module("basic",
|
||||
"tests/resources/importer/basic.hy")
|
||||
|
||||
basic_namespace = runpy.run_path("tests/resources/importer/basic.hy",
|
||||
run_name='__main__')
|
||||
assert 'square' in basic_namespace
|
||||
|
||||
|
||||
def test_stringer():
|
||||
_ast = import_buffer_to_ast("(defn square [x] (* x x))", '')
|
||||
_ast = hy_compile(hy_parse("(defn square [x] (* x x))"), '')
|
||||
|
||||
assert type(_ast.body[0]) == ast.FunctionDef
|
||||
|
||||
|
||||
def test_imports():
|
||||
path = os.getcwd() + "/tests/resources/importer/a.hy"
|
||||
testLoader = MetaLoader(path)
|
||||
testLoader = HyLoader("tests.resources.importer.a", path)
|
||||
|
||||
def _import_test():
|
||||
try:
|
||||
return testLoader.load_module("tests.resources.importer.a")
|
||||
return testLoader.load_module()
|
||||
except:
|
||||
return "Error"
|
||||
|
||||
@ -43,7 +51,7 @@ def test_import_error_reporting():
|
||||
|
||||
def _import_error_test():
|
||||
try:
|
||||
import_buffer_to_ast("(import \"sys\")", '')
|
||||
_ = hy_compile(hy_parse("(import \"sys\")"), '')
|
||||
except HyTypeError:
|
||||
return "Error reported"
|
||||
|
||||
@ -56,19 +64,23 @@ def test_import_error_reporting():
|
||||
def test_import_autocompiles():
|
||||
"Test that (import) byte-compiles the module."
|
||||
|
||||
f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False)
|
||||
with tempfile.NamedTemporaryFile(suffix='.hy', delete=True) as f:
|
||||
f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
|
||||
f.close()
|
||||
f.flush()
|
||||
|
||||
pyc_path = cache_from_source(f.name)
|
||||
|
||||
try:
|
||||
os.remove(get_bytecode_path(f.name))
|
||||
os.remove(pyc_path)
|
||||
except (IOError, OSError):
|
||||
pass
|
||||
import_file_to_module("mymodule", f.name)
|
||||
assert os.path.exists(get_bytecode_path(f.name))
|
||||
|
||||
os.remove(f.name)
|
||||
os.remove(get_bytecode_path(f.name))
|
||||
test_loader = HyLoader("mymodule", f.name).load_module()
|
||||
|
||||
assert hasattr(test_loader, 'pyctest')
|
||||
assert os.path.exists(pyc_path)
|
||||
|
||||
os.remove(pyc_path)
|
||||
|
||||
|
||||
def test_eval():
|
||||
|
||||
@ -5,19 +5,20 @@
|
||||
import os
|
||||
import imp
|
||||
import tempfile
|
||||
from hy.importer import write_hy_as_pyc, get_bytecode_path
|
||||
|
||||
import py_compile
|
||||
|
||||
|
||||
def test_pyc():
|
||||
"""Test pyc compilation."""
|
||||
f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False)
|
||||
with tempfile.NamedTemporaryFile(suffix='.hy') as f:
|
||||
f.write(b'(defn pyctest [s] (+ "X" s "Y"))')
|
||||
f.close()
|
||||
f.flush()
|
||||
|
||||
write_hy_as_pyc(f.name)
|
||||
os.remove(f.name)
|
||||
cfile = py_compile.compile(f.name)
|
||||
|
||||
assert os.path.exists(cfile)
|
||||
|
||||
cfile = get_bytecode_path(f.name)
|
||||
mod = imp.load_compiled('pyc', cfile)
|
||||
os.remove(cfile)
|
||||
|
||||
|
||||
@ -147,7 +147,7 @@
|
||||
(defn test-gensym-in-macros []
|
||||
(import ast)
|
||||
(import [astor.code-gen [to-source]])
|
||||
(import [hy.importer [import_buffer_to_ast]])
|
||||
(import [hy.importer [hy-parse hy-compile]])
|
||||
(setv macro1 "(defmacro nif [expr pos zero neg]
|
||||
(setv g (gensym))
|
||||
`(do
|
||||
@ -160,8 +160,8 @@
|
||||
")
|
||||
;; expand the macro twice, should use a different
|
||||
;; gensym each time
|
||||
(setv _ast1 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast2 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast1 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv _ast2 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv s1 (to_source _ast1))
|
||||
(setv s2 (to_source _ast2))
|
||||
;; and make sure there is something new that starts with _;G|
|
||||
@ -173,7 +173,7 @@
|
||||
(defn test-with-gensym []
|
||||
(import ast)
|
||||
(import [astor.code-gen [to-source]])
|
||||
(import [hy.importer [import_buffer_to_ast]])
|
||||
(import [hy.importer [hy-parse hy-compile]])
|
||||
(setv macro1 "(defmacro nif [expr pos zero neg]
|
||||
(with-gensyms [a]
|
||||
`(do
|
||||
@ -186,8 +186,8 @@
|
||||
")
|
||||
;; expand the macro twice, should use a different
|
||||
;; gensym each time
|
||||
(setv _ast1 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast2 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast1 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv _ast2 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv s1 (to_source _ast1))
|
||||
(setv s2 (to_source _ast2))
|
||||
(assert (in (mangle "_;a|") s1))
|
||||
@ -197,7 +197,7 @@
|
||||
(defn test-defmacro/g! []
|
||||
(import ast)
|
||||
(import [astor.code-gen [to-source]])
|
||||
(import [hy.importer [import_buffer_to_ast]])
|
||||
(import [hy.importer [hy-parse hy-compile]])
|
||||
(setv macro1 "(defmacro/g! nif [expr pos zero neg]
|
||||
`(do
|
||||
(setv ~g!res ~expr)
|
||||
@ -209,8 +209,8 @@
|
||||
")
|
||||
;; expand the macro twice, should use a different
|
||||
;; gensym each time
|
||||
(setv _ast1 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast2 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast1 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv _ast2 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv s1 (to_source _ast1))
|
||||
(setv s2 (to_source _ast2))
|
||||
(assert (in (mangle "_;res|") s1))
|
||||
@ -220,13 +220,13 @@
|
||||
;; defmacro/g! didn't like numbers initially because they
|
||||
;; don't have a startswith method and blew up during expansion
|
||||
(setv macro2 "(defmacro/g! two-point-zero [] `(+ (float 1) 1.0))")
|
||||
(assert (import_buffer_to_ast macro2 "foo")))
|
||||
(assert (hy-compile (hy-parse macro2) "foo")))
|
||||
|
||||
(defn test-defmacro! []
|
||||
;; defmacro! must do everything defmacro/g! can
|
||||
(import ast)
|
||||
(import [astor.code-gen [to-source]])
|
||||
(import [hy.importer [import_buffer_to_ast]])
|
||||
(import [hy.importer [hy-parse hy-compile]])
|
||||
(setv macro1 "(defmacro! nif [expr pos zero neg]
|
||||
`(do
|
||||
(setv ~g!res ~expr)
|
||||
@ -238,8 +238,8 @@
|
||||
")
|
||||
;; expand the macro twice, should use a different
|
||||
;; gensym each time
|
||||
(setv _ast1 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast2 (import_buffer_to_ast macro1 "foo"))
|
||||
(setv _ast1 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv _ast2 (hy-compile (hy-parse macro1) "foo"))
|
||||
(setv s1 (to_source _ast1))
|
||||
(setv s2 (to_source _ast2))
|
||||
(assert (in (mangle "_;res|") s1))
|
||||
@ -249,7 +249,7 @@
|
||||
;; defmacro/g! didn't like numbers initially because they
|
||||
;; don't have a startswith method and blew up during expansion
|
||||
(setv macro2 "(defmacro! two-point-zero [] `(+ (float 1) 1.0))")
|
||||
(assert (import_buffer_to_ast macro2 "foo"))
|
||||
(assert (hy-compile (hy-parse macro2) "foo"))
|
||||
|
||||
(defmacro! foo! [o!foo] `(do ~g!foo ~g!foo))
|
||||
;; test that o! becomes g!
|
||||
|
||||
@ -9,10 +9,11 @@ import re
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from hy.importer import cache_from_source
|
||||
|
||||
import pytest
|
||||
|
||||
from hy._compat import builtins
|
||||
from hy.importer import get_bytecode_path
|
||||
|
||||
|
||||
hy_dir = os.environ.get('HY_DIR', '')
|
||||
@ -231,8 +232,11 @@ def test_bin_hy_file_with_args():
|
||||
|
||||
|
||||
def test_bin_hyc():
|
||||
_, err = run_cmd("hyc", expect=2)
|
||||
assert "usage" in err
|
||||
_, err = run_cmd("hyc", expect=0)
|
||||
assert err == ''
|
||||
|
||||
_, err = run_cmd("hyc -", expect=0)
|
||||
assert err == ''
|
||||
|
||||
output, _ = run_cmd("hyc -h")
|
||||
assert "usage" in output
|
||||
@ -240,12 +244,12 @@ def test_bin_hyc():
|
||||
path = "tests/resources/argparse_ex.hy"
|
||||
output, _ = run_cmd("hyc " + path)
|
||||
assert "Compiling" in output
|
||||
assert os.path.exists(get_bytecode_path(path))
|
||||
rm(get_bytecode_path(path))
|
||||
assert os.path.exists(cache_from_source(path))
|
||||
rm(cache_from_source(path))
|
||||
|
||||
|
||||
def test_bin_hyc_missing_file():
|
||||
_, err = run_cmd("hyc foobarbaz", expect=2)
|
||||
_, err = run_cmd("hyc foobarbaz", expect=1)
|
||||
assert "[Errno 2]" in err
|
||||
|
||||
|
||||
@ -284,19 +288,19 @@ def test_bin_hy_no_main():
|
||||
@pytest.mark.parametrize('scenario', [
|
||||
"normal", "prevent_by_force", "prevent_by_env"])
|
||||
@pytest.mark.parametrize('cmd_fmt', [
|
||||
'hy {fpath}', 'hy -m {modname}', "hy -c '(import {modname})'"])
|
||||
'hy -m {modname}', "hy -c '(import {modname})'"])
|
||||
def test_bin_hy_byte_compile(scenario, cmd_fmt):
|
||||
|
||||
modname = "tests.resources.bin.bytecompile"
|
||||
fpath = modname.replace(".", "/") + ".hy"
|
||||
cmd = cmd_fmt.format(**locals())
|
||||
|
||||
rm(get_bytecode_path(fpath))
|
||||
rm(cache_from_source(fpath))
|
||||
|
||||
if scenario == "prevent_by_force":
|
||||
# Keep Hy from being able to byte-compile the module by
|
||||
# creating a directory at the target location.
|
||||
os.mkdir(get_bytecode_path(fpath))
|
||||
os.mkdir(cache_from_source(fpath))
|
||||
|
||||
# Whether or not we can byte-compile the module, we should be able
|
||||
# to run it.
|
||||
@ -306,10 +310,10 @@ def test_bin_hy_byte_compile(scenario, cmd_fmt):
|
||||
|
||||
if scenario == "normal":
|
||||
# That should've byte-compiled the module.
|
||||
assert os.path.exists(get_bytecode_path(fpath))
|
||||
assert os.path.exists(cache_from_source(fpath))
|
||||
elif scenario == "prevent_by_env":
|
||||
# No byte-compiled version should've been created.
|
||||
assert not os.path.exists(get_bytecode_path(fpath))
|
||||
assert not os.path.exists(cache_from_source(fpath))
|
||||
|
||||
# When we run the same command again, and we've byte-compiled the
|
||||
# module, the byte-compiled version should be run instead of the
|
||||
|
||||
Loading…
Reference in New Issue
Block a user