From 87a5b117a14f09f3ca0b35135d1e3c175b124bae Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Sun, 19 Aug 2018 23:29:29 -0500 Subject: [PATCH] Implement new importer using PEP-302 semantics Python 3.x is patched in a way that integrates `.hy` source files into Pythons default `importlib` machinery. In Python 2.7, a PEP-302 "importer" and "loader" is implemented according to the standard `import` logic (via `pkgutil` and later pure-Python `imp` package code). In both cases, the entry-point for the loaders is through `sys.path_hooks` only. As well, the import semantics have been updated all throughout to utilize `importlib` and follow aspects of PEP-420. This, along with some light patches, should allow for basic use of `runpy`, `py_compile` and `reload`. In all cases, if a `.hy` file is shadowed by a `.py`, Hy will silently use `.hy`. --- conftest.py | 49 +- docs/contrib/walk.rst | 2 +- hy/_compat.py | 17 +- hy/cmdline.py | 117 +++-- hy/compiler.py | 2 +- hy/contrib/walk.hy | 2 +- hy/core/macros.hy | 9 +- hy/importer.py | 705 +++++++++++++++++++--------- hy/macros.py | 20 +- tests/compilers/test_ast.py | 10 +- tests/importer/test_importer.py | 54 ++- tests/importer/test_pyc.py | 21 +- tests/native_tests/native_macros.hy | 28 +- tests/test_bin.py | 26 +- 14 files changed, 691 insertions(+), 371 deletions(-) diff --git a/conftest.py b/conftest.py index 54ba2e6..5b1c41d 100644 --- a/conftest.py +++ b/conftest.py @@ -1,15 +1,50 @@ +import os +import importlib + +import py import pytest import hy -import os from hy._compat import PY3, PY35, PY36 NATIVE_TESTS = os.path.join("", "tests", "native_tests", "") +_fspath_pyimport = py.path.local.pyimport + + +def pytest_ignore_collect(path, config): + return (("py3_only" in path.basename and not PY3) + or ("py35_only" in path.basename and not PY35) + or ("py36_only" in path.basename and not PY36)) + + +def pyimport_patch_mismatch(self, **kwargs): + """Lame fix for https://github.com/pytest-dev/py/issues/195""" + try: + return _fspath_pyimport(self, **kwargs) + except py.path.local.ImportMismatchError: + pkgpath = self.pypkgpath() + if pkgpath is None: + pkgroot = self.dirpath() + modname = self.purebasename + else: + pkgroot = pkgpath.dirpath() + names = self.new(ext="").relto(pkgroot).split(self.sep) + if names[-1] == "__init__": + names.pop() + modname = ".".join(names) + + res = importlib.import_module(modname) + + return res + + +py.path.local.pyimport = pyimport_patch_mismatch + + def pytest_collect_file(parent, path): if (path.ext == ".hy" - and NATIVE_TESTS in path.dirname + os.sep - and path.basename != "__init__.hy" - and not ("py3_only" in path.basename and not PY3) - and not ("py35_only" in path.basename and not PY35) - and not ("py36_only" in path.basename and not PY36)): - return pytest.Module(path, parent) + and NATIVE_TESTS in path.dirname + os.sep + and path.basename != "__init__.hy"): + + pytest_mod = pytest.Module(path, parent) + return pytest_mod diff --git a/docs/contrib/walk.rst b/docs/contrib/walk.rst index 1bd0fcc..2e36234 100644 --- a/docs/contrib/walk.rst +++ b/docs/contrib/walk.rst @@ -231,7 +231,7 @@ But assignments via ``import`` are always hoisted to normal Python scope, and likewise, ``defclass`` will assign the class to the Python scope, even if it shares the name of a let binding. -Use ``__import__`` and ``type`` (or whatever metaclass) instead, +Use ``importlib.import_module`` and ``type`` (or whatever metaclass) instead, if you must avoid this hoisting. The ``let`` macro takes two parameters: a list defining *variables* diff --git a/hy/_compat.py b/hy/_compat.py index c40e44d..4416ea5 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -6,18 +6,6 @@ try: import __builtin__ as builtins except ImportError: import builtins # NOQA -try: - from py_compile import MAGIC, wr_long -except ImportError: - # py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4 - from importlib.util import MAGIC_NUMBER as MAGIC # NOQA - - def wr_long(f, x): - """Internal; write a 32-bit int to a file in little-endian order.""" - f.write(bytes([x & 0xff, - (x >> 8) & 0xff, - (x >> 16) & 0xff, - (x >> 24) & 0xff])) import sys, keyword PY3 = sys.version_info[0] >= 3 @@ -60,3 +48,8 @@ def isidentifier(x): except T.TokenError: return False return len(tokens) == 2 and tokens[0][0] == T.NAME + +try: + FileNotFoundError = FileNotFoundError +except NameError: + FileNotFoundError = IOError diff --git a/hy/cmdline.py b/hy/cmdline.py index b10028c..f1c7a9e 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -9,26 +9,21 @@ import code import ast import sys import os +import io import importlib +import py_compile +import runpy import astor.code_gen import hy - from hy.lex import LexException, PrematureEndOfInput, mangle -from hy.compiler import HyTypeError -from hy.importer import (hy_eval, import_buffer_to_module, - import_file_to_ast, import_file_to_hst, - import_buffer_to_ast, import_buffer_to_hst) -from hy.completer import completion -from hy.completer import Completer - -from hy.errors import HyIOError - +from hy.compiler import HyTypeError, hy_compile +from hy.importer import hy_eval, hy_parse +from hy.completer import completion, Completer from hy.macros import macro, require from hy.models import HyExpression, HyString, HySymbol - -from hy._compat import builtins, PY3 +from hy._compat import builtins, PY3, FileNotFoundError class HyQuitter(object): @@ -47,6 +42,7 @@ class HyQuitter(object): pass raise SystemExit(code) + builtins.quit = HyQuitter('quit') builtins.exit = HyQuitter('exit') @@ -88,7 +84,7 @@ class HyREPL(code.InteractiveConsole): try: try: - do = import_buffer_to_hst(source) + do = hy_parse(source) except PrematureEndOfInput: return True except LexException as e: @@ -202,25 +198,8 @@ def pretty_error(func, *args, **kw): def run_command(source): - pretty_error(import_buffer_to_module, "__main__", source) - return 0 - - -def run_module(mod_name): - from hy.importer import MetaImporter - pth = MetaImporter().find_on_path(mod_name) - if pth is not None: - sys.argv = [pth] + sys.argv - return run_file(pth) - - print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name), - file=sys.stderr) - return 1 - - -def run_file(filename): - from hy.importer import import_file_to_module - pretty_error(import_file_to_module, "__main__", filename) + tree = hy_parse(source) + pretty_error(hy_eval, tree, module_name="__main__") return 0 @@ -252,7 +231,7 @@ def run_repl(hr=None, **kwargs): def run_icommand(source, **kwargs): hr = HyREPL(**kwargs) if os.path.exists(source): - with open(source, "r") as f: + with io.open(source, "r", encoding='utf-8') as f: source = f.read() filename = source else: @@ -320,7 +299,7 @@ def cmdline_handler(scriptname, argv): SIMPLE_TRACEBACKS = False # reset sys.argv like Python - sys.argv = options.args + module_args or [""] + # sys.argv = [sys.argv[0]] + options.args + module_args if options.E: # User did "hy -E ..." @@ -332,7 +311,9 @@ def cmdline_handler(scriptname, argv): if options.mod: # User did "hy -m ..." - return run_module(options.mod) + sys.argv = [sys.argv[0]] + options.args + module_args + runpy.run_module(options.mod, run_name='__main__', alter_sys=True) + return 0 if options.icommand: # User did "hy -i ..." @@ -347,10 +328,12 @@ def cmdline_handler(scriptname, argv): else: # User did "hy " try: - return run_file(options.args[0]) - except HyIOError as e: - print("hy: Can't open file '{0}': [Errno {1}] {2}\n".format( - e.filename, e.errno, e.strerror), file=sys.stderr) + sys.argv = options.args + runpy.run_path(options.args[0], run_name='__main__') + return 0 + except FileNotFoundError as e: + print("hy: Can't open file '{0}': [Errno {1}] {2}".format( + e.filename, e.errno, e.strerror), file=sys.stderr) sys.exit(e.errno) # User did NOTHING! @@ -359,27 +342,45 @@ def cmdline_handler(scriptname, argv): # entry point for cmd line script "hy" def hy_main(): + sys.path.insert(0, "") sys.exit(cmdline_handler("hy", sys.argv)) -# entry point for cmd line script "hyc" def hyc_main(): - from hy.importer import write_hy_as_pyc parser = argparse.ArgumentParser(prog="hyc") - parser.add_argument("files", metavar="FILE", nargs='+', - help="file to compile") + parser.add_argument("files", metavar="FILE", nargs='*', + help=('File(s) to compile (use STDIN if only' + ' "-" or nothing is provided)')) parser.add_argument("-v", action="version", version=VERSION) options = parser.parse_args(sys.argv[1:]) - for file in options.files: - try: - print("Compiling %s" % file) - pretty_error(write_hy_as_pyc, file) - except IOError as x: - print("hyc: Can't open file '{0}': [Errno {1}] {2}\n".format( - x.filename, x.errno, x.strerror), file=sys.stderr) - sys.exit(x.errno) + rv = 0 + if len(options.files) == 0 or ( + len(options.files) == 1 and options.files[0] == '-'): + while True: + filename = sys.stdin.readline() + if not filename: + break + filename = filename.rstrip('\n') + try: + py_compile.compile(filename, doraise=True) + except py_compile.PyCompileError as error: + rv = 1 + sys.stderr.write("%s\n" % error.msg) + except OSError as error: + rv = 1 + sys.stderr.write("%s\n" % error) + else: + for filename in options.files: + try: + print("Compiling %s" % filename) + py_compile.compile(filename, doraise=True) + except py_compile.PyCompileError as error: + # return value to indicate at least one failure + rv = 1 + sys.stderr.write("%s\n" % error.msg) + return rv # entry point for cmd line script "hy2py" @@ -403,14 +404,14 @@ def hy2py_main(): options = parser.parse_args(sys.argv[1:]) - stdin_text = None if options.FILE is None or options.FILE == '-': - stdin_text = sys.stdin.read() + source = sys.stdin.read() + else: + with io.open(options.FILE, 'r', encoding='utf-8') as source_file: + source = source_file.read() + hst = pretty_error(hy_parse, source) if options.with_source: - hst = (pretty_error(import_file_to_hst, options.FILE) - if stdin_text is None - else pretty_error(import_buffer_to_hst, stdin_text)) # need special printing on Windows in case the # codepage doesn't support utf-8 characters if PY3 and platform.system() == "Windows": @@ -424,9 +425,7 @@ def hy2py_main(): print() print() - _ast = (pretty_error(import_file_to_ast, options.FILE, module_name) - if stdin_text is None - else pretty_error(import_buffer_to_ast, stdin_text, module_name)) + _ast = pretty_error(hy_compile, hst, module_name) if options.with_ast: if PY3 and platform.system() == "Windows": _print_for_windows(astor.dump_tree(_ast)) diff --git a/hy/compiler.py b/hy/compiler.py index f25f615..575c5a9 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -1151,7 +1151,7 @@ class HyASTCompiler(object): ret += node( expr, module=module or None, names=names, level=level) else: # root == "require" - __import__(module) + importlib.import_module(module) require(module, self.module_name, assignments=assignments, prefix=prefix) diff --git a/hy/contrib/walk.hy b/hy/contrib/walk.hy index 4d91721..4b6df7a 100644 --- a/hy/contrib/walk.hy +++ b/hy/contrib/walk.hy @@ -299,7 +299,7 @@ But assignments via `import` are always hoisted to normal Python scope, and likewise, `defclass` will assign the class to the Python scope, even if it shares the name of a let binding. -Use __import__ and type (or whatever metaclass) instead, +Use `import_module` and `type` (or whatever metaclass) instead, if you must avoid this hoisting. Function arguments can shadow let bindings in their body, diff --git a/hy/core/macros.hy b/hy/core/macros.hy index c607c74..744e237 100644 --- a/hy/core/macros.hy +++ b/hy/core/macros.hy @@ -6,6 +6,7 @@ ;;; These macros form the hy language ;;; They are automatically required in every module, except inside hy.core +(import [importlib [import-module]]) (import [hy.models [HyList HySymbol]]) @@ -247,13 +248,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol." Use ``#doc foo`` instead for help with tag macro ``#foo``. Use ``(help foo)`` instead for help with runtime objects." `(try - (help (. (__import__ "hy") + (help (. (import-module "hy") macros _hy_macros [__name__] ['~symbol])) (except [KeyError] - (help (. (__import__ "hy") + (help (. (import-module "hy") macros _hy_macros [None] @@ -264,13 +265,13 @@ Such 'o!' params are available within `body` as the equivalent 'g!' symbol." Gets help for a tag macro function available in this module." `(try - (help (. (__import__ "hy") + (help (. (import-module "hy") macros _hy_tag [__name__] ['~symbol])) (except [KeyError] - (help (. (__import__ "hy") + (help (. (import-module "hy") macros _hy_tag [None] diff --git a/hy/importer.py b/hy/importer.py index a93d1fd..390f373 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -4,158 +4,75 @@ from __future__ import absolute_import -from hy.compiler import hy_compile, HyTypeError -from hy.models import HyExpression, HySymbol -from hy.lex import tokenize, LexException -from hy.errors import HyIOError - -from io import open -import re -import marshal -import struct -import imp import sys +import os import ast import inspect -import os +import pkgutil +import re +import io +import runpy +import types +import tempfile +import importlib import __future__ -from hy._compat import PY3, PY37, MAGIC, builtins, long_type, wr_long -from hy._compat import string_types +from hy.errors import HyTypeError +from hy.compiler import hy_compile +from hy.lex import tokenize, LexException +from hy.models import HyExpression, HySymbol +from hy._compat import string_types, PY3 + + +hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION | + __future__.CO_FUTURE_PRINT_FUNCTION) def ast_compile(ast, filename, mode): """Compile AST. - Like Python's compile, but with some special flags.""" - flags = (__future__.CO_FUTURE_DIVISION | - __future__.CO_FUTURE_PRINT_FUNCTION) - return compile(ast, filename, mode, flags) + + Parameters + ---------- + ast : instance of `ast.AST` + + filename : str + Filename used for run-time error messages + + mode: str + `compile` mode parameter + + Returns + ------- + out : instance of `types.CodeType` + """ + return compile(ast, filename, mode, hy_ast_compile_flags) -def import_buffer_to_hst(buf): - """Import content from buf and return a Hy AST.""" - return HyExpression([HySymbol("do")] + tokenize(buf + "\n")) +def hy_parse(source): + """Parse a Hy source string. + Parameters + ---------- + source: string + Source code to parse. -def import_file_to_hst(fpath): - """Import content from fpath and return a Hy AST.""" - try: - with open(fpath, 'r', encoding='utf-8') as f: - buf = f.read() - # Strip the shebang line, if there is one. - buf = re.sub(r'\A#!.*', '', buf) - return import_buffer_to_hst(buf) - except IOError as e: - raise HyIOError(e.errno, e.strerror, e.filename) - - -def import_buffer_to_ast(buf, module_name): - """ Import content from buf and return a Python AST.""" - return hy_compile(import_buffer_to_hst(buf), module_name) - - -def import_file_to_ast(fpath, module_name): - """Import content from fpath and return a Python AST.""" - return hy_compile(import_file_to_hst(fpath), module_name) - - -def import_file_to_module(module_name, fpath, loader=None): - """Import Hy source from fpath and put it into a Python module. - - If there's an up-to-date byte-compiled version of this module, load that - instead. Otherwise, byte-compile the module once we're done loading it, if - we can. - - Return the module.""" - - module = None - - bytecode_path = get_bytecode_path(fpath) - try: - source_mtime = int(os.stat(fpath).st_mtime) - with open(bytecode_path, 'rb') as bc_f: - # The first 4 bytes are the magic number for the version of Python - # that compiled this bytecode. - bytecode_magic = bc_f.read(4) - # Python 3.7 introduced a new flags entry in the header structure. - if PY37: - bc_f.read(4) - # The next 4 bytes, interpreted as a little-endian 32-bit integer, - # are the mtime of the corresponding source file. - bytecode_mtime, = struct.unpack('= source_mtime: - # It's a cache hit. Load the byte-compiled version. - if PY3: - # As of Python 3.6, imp.load_compiled still exists, but it's - # deprecated. So let's use SourcelessFileLoader instead. - from importlib.machinery import SourcelessFileLoader - module = (SourcelessFileLoader(module_name, bytecode_path). - load_module(module_name)) - else: - module = imp.load_compiled(module_name, bytecode_path) - - if not module: - # It's a cache miss, so load from source. - sys.modules[module_name] = None - try: - _ast = import_file_to_ast(fpath, module_name) - module = imp.new_module(module_name) - module.__file__ = os.path.normpath(fpath) - code = ast_compile(_ast, fpath, "exec") - if not os.environ.get('PYTHONDONTWRITEBYTECODE'): - try: - write_code_as_pyc(fpath, code) - except (IOError, OSError): - # We failed to save the bytecode, probably because of a - # permissions issue. The user only asked to import the - # file, so don't bug them about it. - pass - eval(code, module.__dict__) - except (HyTypeError, LexException) as e: - if e.source is None: - with open(fpath, 'rt') as fp: - e.source = fp.read() - e.filename = fpath - raise - except Exception: - sys.modules.pop(module_name, None) - raise - sys.modules[module_name] = module - module.__name__ = module_name - - module.__file__ = os.path.normpath(fpath) - if loader: - module.__loader__ = loader - if is_package(module_name): - module.__path__ = [] - module.__package__ = module_name - else: - module.__package__ = module_name.rpartition('.')[0] - - return module - - -def import_buffer_to_module(module_name, buf): - try: - _ast = import_buffer_to_ast(buf, module_name) - mod = imp.new_module(module_name) - eval(ast_compile(_ast, "", "exec"), mod.__dict__) - except (HyTypeError, LexException) as e: - if e.source is None: - e.source = buf - e.filename = '' - raise - return mod + Returns + ------- + out : instance of `types.CodeType` + """ + source = re.sub(r'\A#!.*', '', source) + return HyExpression([HySymbol("do")] + tokenize(source + "\n")) def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): - """``eval`` evaluates a quoted expression and returns the value. The optional - second and third arguments specify the dictionary of globals to use and the - module name. The globals dictionary defaults to ``(local)`` and the module - name defaults to the name of the current module. + """Evaluates a quoted expression and returns the value. + + The optional second and third arguments specify the dictionary of globals + to use and the module name. The globals dictionary defaults to ``(local)`` + and the module name defaults to the name of the current module. + + Examples + -------- => (eval '(print "Hello World")) "Hello World" @@ -164,7 +81,31 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): form first: => (eval (read-str "(+ 1 1)")) - 2""" + 2 + + Parameters + ---------- + hytree: a Hy expression tree + Source code to parse. + + namespace: dict, optional + Namespace in which to evaluate the Hy tree. Defaults to the calling + frame. + + module_name: str, optional + Name of the module to which the Hy tree is assigned. Defaults to + the calling frame's module, if any, and '__eval__' otherwise. + + ast_callback: callable, optional + A callback that is passed the Hy compiled tree and resulting + expression object, in that order, after compilation but before + evaluation. + + Returns + ------- + out : Result of evaluating the Hy compiled tree. + + """ if namespace is None: frame = inspect.stack()[1][0] namespace = inspect.getargvalues(frame).locals @@ -199,89 +140,427 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): return eval(ast_compile(expr, "", "eval"), namespace) -def write_hy_as_pyc(fname): - _ast = import_file_to_ast(fname, - os.path.basename(os.path.splitext(fname)[0])) - code = ast_compile(_ast, fname, "exec") - write_code_as_pyc(fname, code) +def cache_from_source(source_path): + """Get the cached bytecode file name for a given source file name. + This function's name is set to mirror Python 3.x's + `importlib.util.cache_from_source`, which is also used when available. -def write_code_as_pyc(fname, code): - st = os.stat(fname) - timestamp = long_type(st.st_mtime) + Parameters + ---------- + source_path : str + Path of the source file - cfile = get_bytecode_path(fname) - try: - os.makedirs(os.path.dirname(cfile)) - except (IOError, OSError): - pass - - with builtins.open(cfile, 'wb') as fc: - fc.write(MAGIC) - if PY37: - # With PEP 552, the header structure has a new flags field - # that we need to fill in. All zeros preserve the legacy - # behaviour, but should we implement reproducible builds, - # this is where we'd add the information. - wr_long(fc, 0) - wr_long(fc, timestamp) - if PY3: - wr_long(fc, st.st_size) - marshal.dump(code, fc) - - -class MetaLoader(object): - def __init__(self, path): - self.path = path - - def load_module(self, fullname): - if fullname in sys.modules: - return sys.modules[fullname] - - if not self.path: - return - - return import_file_to_module(fullname, self.path, self) - - -class MetaImporter(object): - def find_on_path(self, fullname): - fls = ["%s/__init__.hy", "%s.hy"] - dirpath = "/".join(fullname.split(".")) - - for pth in sys.path: - pth = os.path.abspath(pth) - for fp in fls: - composed_path = fp % ("%s/%s" % (pth, dirpath)) - if os.path.exists(composed_path): - return composed_path - - def find_module(self, fullname, path=None): - path = self.find_on_path(fullname) - if path: - return MetaLoader(path) - - -sys.meta_path.insert(0, MetaImporter()) -sys.path.insert(0, "") - - -def is_package(module_name): - mpath = os.path.join(*module_name.split(".")) - for path in map(os.path.abspath, sys.path): - if os.path.exists(os.path.join(path, mpath, "__init__.hy")): - return True - return False - - -def get_bytecode_path(source_path): + Returns + ------- + out : str + Path of the corresponding bytecode file that may--or may + not--actually exist. + """ if PY3: - import importlib.util return importlib.util.cache_from_source(source_path) - elif hasattr(imp, "cache_from_source"): - return imp.cache_from_source(source_path) else: # If source_path has a file extension, replace it with ".pyc". # Otherwise, just append ".pyc". d, f = os.path.split(source_path) return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f)) + + +def _get_code_from_file(run_name, fname=None): + """A patch of `runpy._get_code_from_file` that will also compile Hy + code. + + This version will read and cache bytecode for Hy files. It operates + normally otherwise. + """ + if fname is None and run_name is not None: + fname = run_name + + if fname.endswith('.hy'): + full_fname = os.path.abspath(fname) + fname_path, fname_file = os.path.split(full_fname) + modname = os.path.splitext(fname_file)[0] + sys.path.insert(0, fname_path) + try: + loader = pkgutil.get_loader(modname) + code = loader.get_code(modname) + finally: + sys.path.pop(0) + else: + with open(fname, "rb") as f: + code = pkgutil.read_code(f) + if code is None: + with open(fname, "rb") as f: + source = f.read().decode('utf-8') + code = compile(source, fname, 'exec') + + return (code, fname) if PY3 else code + + +_runpy_get_code_from_file = runpy._get_code_from_file +runpy._get_code_from_file = _get_code_from_file + +if PY3: + importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy') + _py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code + + def _hy_source_to_code(self, data, path, _optimize=-1): + if os.path.isfile(path) and path.endswith('.hy'): + source = data.decode("utf-8") + try: + hy_tree = hy_parse(source) + data = hy_compile(hy_tree, self.name) + except (HyTypeError, LexException) as e: + if e.source is None: + e.source = source + e.filename = path + raise + + return _py_source_to_code(self, data, path, _optimize=_optimize) + + importlib.machinery.SourceFileLoader.source_to_code = _hy_source_to_code + + # This is actually needed; otherwise, pre-created finders assigned to the + # current dir (i.e. `''`) in `sys.path` will not catch absolute imports of + # directory-local modules! + sys.path_importer_cache.clear() + + # Do this one just in case? + importlib.invalidate_caches() + + # XXX: These and the 2.7 counterparts below aren't truly cross-compliant. + # They're useful for testing, though. + HyImporter = importlib.machinery.FileFinder + HyLoader = importlib.machinery.SourceFileLoader + +else: + import imp + import py_compile + import marshal + import struct + import traceback + + from pkgutil import ImpImporter, ImpLoader + + # 100x better! + HY_SOURCE = imp.PY_SOURCE * 100 + + class HyLoader(ImpLoader, object): + def __init__(self, fullname, filename, fileobj=None, etc=None): + """This constructor is designed for some compatibility with + SourceFileLoader.""" + if etc is None and filename is not None: + if filename.endswith('.hy'): + etc = ('.hy', 'U', HY_SOURCE) + if fileobj is None: + fileobj = open(filename, 'rU') + + super(HyLoader, self).__init__(fullname, fileobj, filename, etc) + + def exec_module(self, module, fullname=None): + fullname = self._fix_name(fullname) + ast = self.get_code(fullname) + eval(ast, module.__dict__) + + def load_module(self, fullname=None): + """Same as `pkgutil.ImpLoader`, with an extra check for Hy + source""" + fullname = self._fix_name(fullname) + mod_type = self.etc[2] + mod = None + pkg_path = os.path.join(self.filename, '__init__.hy') + if mod_type == HY_SOURCE or ( + mod_type == imp.PKG_DIRECTORY and + os.path.isfile(pkg_path)): + + if fullname in sys.modules: + mod = sys.modules[fullname] + else: + mod = sys.modules.setdefault( + fullname, imp.new_module(fullname)) + + # TODO: Should we set these only when not in `sys.modules`? + if mod_type == imp.PKG_DIRECTORY: + mod.__file__ = pkg_path + mod.__path__ = [self.filename] + mod.__package__ = fullname + else: + # mod.__path__ = self.filename + mod.__file__ = self.get_filename(fullname) + mod.__package__ = '.'.join(fullname.split('.')[:-1]) + + # TODO: Set `mod.__doc__`. + mod.__name__ = fullname + + self.exec_module(mod, fullname=fullname) + + if mod is None: + self._reopen() + try: + mod = imp.load_module(fullname, self.file, self.filename, + self.etc) + finally: + if self.file: + self.file.close() + + mod.__loader__ = self + return mod + + def _reopen(self): + """Same as `pkgutil.ImpLoader`, with an extra check for Hy + source""" + super(HyLoader, self)._reopen() + + # Add the Hy case... + if self.file and self.file.closed: + mod_type = self.etc[2] + if mod_type == HY_SOURCE: + self.file = io.open(self.filename, 'rU', encoding='utf-8') + + def byte_compile_hy(self, fullname=None): + fullname = self._fix_name(fullname) + if fullname is None: + fullname = self.fullname + try: + hy_source = self.get_source(fullname) + hy_tree = hy_parse(hy_source) + ast = hy_compile(hy_tree, fullname) + code = compile(ast, self.filename, 'exec', + hy_ast_compile_flags) + except (HyTypeError, LexException) as e: + if e.source is None: + e.source = hy_source + e.filename = self.filename + raise + + if not sys.dont_write_bytecode: + try: + hyc_compile(code) + except IOError: + pass + return code + + def get_code(self, fullname=None): + """Same as `pkgutil.ImpLoader`, with an extra check for Hy + source""" + fullname = self._fix_name(fullname) + mod_type = self.etc[2] + if mod_type == HY_SOURCE: + # Looks like we have to manually check for--and update-- + # the bytecode. + t_py = long(os.stat(self.filename).st_mtime) + pyc_file = cache_from_source(self.filename) + if os.path.isfile(pyc_file): + t_pyc = long(os.stat(pyc_file).st_mtime) + + if t_pyc is not None and t_pyc >= t_py: + with open(pyc_file, 'rb') as f: + if f.read(4) == imp.get_magic(): + t = struct.unpack('') + flags = hy_ast_compile_flags + + codeobject = compile(source, dfile or filename, 'exec', flags) + except Exception as err: + if isinstance(err, (HyTypeError, LexException)) and err.source is None: + err.source = source_str + err.filename = filename + + py_exc = py_compile.PyCompileError(err.__class__, err, + dfile or filename) + if doraise: + raise py_exc + else: + traceback.print_exc() + return + + timestamp = long(os.stat(filename).st_mtime) + + if cfile is None: + cfile = cache_from_source(filename) + + f = tempfile.NamedTemporaryFile('wb', dir=os.path.split(cfile)[0], + delete=False) + try: + f.write('\0\0\0\0') + f.write(struct.pack('