From 13b484ce460352f74e844f208efb845bacdd7127 Mon Sep 17 00:00:00 2001 From: "Elf M. Sternberg" Date: Sat, 9 Jul 2016 17:47:31 -0700 Subject: [PATCH] Modernized. --- .gitignore | 3 + hy/cmdline.py | 12 +- hy/{importer.py => importer/__init__.py} | 79 ++---- hy/importer/polyloader/__init__.py | 14 ++ hy/importer/polyloader/_python2.py | 188 ++++++++++++++ hy/importer/polyloader/_python3.py | 296 +++++++++++++++++++++++ tests/importer/test_importer.py | 8 +- 7 files changed, 530 insertions(+), 70 deletions(-) rename hy/{importer.py => importer/__init__.py} (76%) create mode 100644 hy/importer/polyloader/__init__.py create mode 100644 hy/importer/polyloader/_python2.py create mode 100644 hy/importer/polyloader/_python3.py diff --git a/.gitignore b/.gitignore index f1cb8d0..d3bb39c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ dist .coverage build/ .noseids +*~ +\#* +.\#* diff --git a/hy/cmdline.py b/hy/cmdline.py index b938df2..8bc13f4 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -196,11 +196,13 @@ def run_command(source): def run_module(mod_name): - from hy.importer import MetaImporter - pth = MetaImporter().find_on_path(mod_name) - if pth is not None: - sys.argv = [pth] + sys.argv - return run_file(pth) + import pkgutil + mod = next((mod for mod in pkgutil.walk_packages() + if mod[1] == mod_name), None) + if mod is not None: + loader = mod[0].find_module(mod_name) + sys.argv = [loader.path] + sys.argv + return run_file(loader.path) print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name), file=sys.stderr) diff --git a/hy/importer.py b/hy/importer/__init__.py similarity index 76% rename from hy/importer.py rename to hy/importer/__init__.py index 8418b7c..826497e 100644 --- a/hy/importer.py +++ b/hy/importer/__init__.py @@ -23,6 +23,7 @@ from hy.compiler import hy_compile, HyTypeError from hy.models import HyObject, replace_hy_obj from hy.lex import tokenize, LexException from hy.errors import HyIOError +from hy.importer import polyloader from io import open import marshal @@ -172,63 +173,23 @@ def write_hy_as_pyc(fname): fc.write(MAGIC) -class MetaLoader(object): - def __init__(self, path): - self.path = path +def _compile_hy(source_text, filename, fullname, *extra): + try: + flags = (__future__.CO_FUTURE_DIVISION | + __future__.CO_FUTURE_PRINT_FUNCTION) + return compile( + hy_compile( + import_buffer_to_hst(source_text.decode('utf-8')), fullname), + filename, "exec", flags) + except (HyTypeError, LexException) as e: + if e.source is None: + with open(filename, 'rt') as fp: + e.source = fp.read() + e.filename = filename + raise + except Exception: + raise - def is_package(self, fullname): - dirpath = "/".join(fullname.split(".")) - for pth in sys.path: - pth = os.path.abspath(pth) - composed_path = "%s/%s/__init__.hy" % (pth, dirpath) - if os.path.exists(composed_path): - return True - return False - - def load_module(self, fullname): - if fullname in sys.modules: - return sys.modules[fullname] - - if not self.path: - return - - sys.modules[fullname] = None - mod = import_file_to_module(fullname, - self.path) - - ispkg = self.is_package(fullname) - - mod.__file__ = self.path - mod.__loader__ = self - mod.__name__ = fullname - - if ispkg: - mod.__path__ = [] - mod.__package__ = fullname - else: - mod.__package__ = fullname.rpartition('.')[0] - - sys.modules[fullname] = mod - return mod - - -class MetaImporter(object): - def find_on_path(self, fullname): - fls = ["%s/__init__.hy", "%s.hy"] - dirpath = "/".join(fullname.split(".")) - - for pth in sys.path: - pth = os.path.abspath(pth) - for fp in fls: - composed_path = fp % ("%s/%s" % (pth, dirpath)) - if os.path.exists(composed_path): - return composed_path - - def find_module(self, fullname, path=None): - path = self.find_on_path(fullname) - if path: - return MetaLoader(path) - - -sys.meta_path.insert(0, MetaImporter()) -sys.path.insert(0, "") +polyloader.install(_compile_hy, ['hy']) +if '' not in sys.path: + sys.path.insert(0, '') diff --git a/hy/importer/polyloader/__init__.py b/hy/importer/polyloader/__init__.py new file mode 100644 index 0000000..26b20c5 --- /dev/null +++ b/hy/importer/polyloader/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +import sys + +__author__ = 'Kenneth M. "Elf" Sternberg' +__email__ = 'elf.sternberg@gmail.com' +__version__ = '0.1.0' + +if sys.version_info[0:2] >= (2, 6): + from ._python2 import install, reset # NOQA + +if sys.version_info[0] >= 3: + from ._python3 import install, reset # NOQA + +__all__ = ['install', 'reset'] diff --git a/hy/importer/polyloader/_python2.py b/hy/importer/polyloader/_python2.py new file mode 100644 index 0000000..dcc1cd8 --- /dev/null +++ b/hy/importer/polyloader/_python2.py @@ -0,0 +1,188 @@ +import io +import os +import os.path +import sys +import imp +import types +import pkgutil +from collections import namedtuple + + +SEP = os.sep +EXS = os.extsep +FLS = [('%s' + SEP + '__init__' + EXS + '%s', True), + ('%s' + EXS + '%s', False)] + +Loader = namedtuple('Loader', 'suffix compiler') + + +class PolyLoader(): + def __init__(self, fullname, path, is_pkg): + self.fullname = fullname + self.path = path + self.is_package = is_pkg + + def load_module(self, fullname): + if fullname in sys.modules: + return sys.modules[fullname] + + if fullname != self.fullname: + raise ImportError("Load confusion: %s vs %s." % + (fullname, self.fullname)) + + matches = [loader for loader in PolyFinder._loader_handlers + if self.path.endswith(loader.suffix)] + + if len(matches) == 0: + raise ImportError("%s is not a recognized module?" % fullname) + + if len(matches) > 1: + raise ImportError("Multiple possible resolutions for %s: %s" % ( + fullname, ', '.join([loader.suffix for loader in matches]))) + + compiler = matches[0].compiler + with io.FileIO(self.path, 'r') as file: + source_text = file.read() + + code = compiler(source_text, self.path, fullname) + + module = types.ModuleType(fullname) + module.__file__ = self.path + module.__name__ = fullname + module.__package__ = '.'.join(fullname.split('.')[:-1]) + + if self.is_package: + module.__path__ = [os.path.dirname(module.__file__)] + module.__package__ = fullname + + exec(code, module.__dict__) + sys.modules[fullname] = module + return module + + +# PolyFinder is an implementation of the Finder class from Python 2.7, +# with embellishments gleefully copied from Python 3.4. It supports +# all the same functionality for non-.py sourcefiles with the added +# benefit of falling back to Python's default behavior. + +# Polyfinder is instantiated by _polyloader_pathhook() + +class PolyFinder(object): + _loader_handlers = [] + _installed = False + + def __init__(self, path=None): + self.path = path or '.' + + def _pl_find_on_path(self, fullname, path=None): + subname = fullname.split(".")[-1] + if self.path is None and subname != fullname: + return None + + path = os.path.realpath(self.path) + for (fp, ispkg) in FLS: + for loader in self._loader_handlers: + composed_path = fp % (('%s' + SEP + '%s') % + (path, subname), loader.suffix) + if os.path.isdir(composed_path): + r = "Invalid: Directory name ends in recognized suffix" + raise IOError(r) + if os.path.isfile(composed_path): + return PolyLoader(fullname, composed_path, ispkg) + + # Fall back onto Python's own methods. + try: + file, filename, etc = imp.find_module(subname, [path]) + except ImportError as e: # NOQA + return None + return pkgutil.ImpLoader(fullname, file, filename, etc) + + def find_module(self, fullname, path=None): + return self._pl_find_on_path(fullname) + + @classmethod + def _install(cls, compiler, suffixes): + if isinstance(suffixes, basestring): + suffixes = [suffixes] + suffixes = set(suffixes) + overlap = suffixes.intersection( + set([suf[0] for suf in imp.get_suffixes()])) + if overlap: + r = "Override of native Python extensions is not permitted." + raise RuntimeError(r) + overlap = suffixes.intersection( + set([loader.suffix for loader in cls._loader_handlers])) + if overlap: + # Fail silently + return + cls._loader_handlers += [Loader(suf, compiler) for suf in suffixes] + + @classmethod + def getmodulename(cls, path): + filename = os.path.basename(path) + suffixes = ([(-len(suf[0]), suf[0]) + for suf in imp.get_suffixes()] + + [(-(len(suf[0]) + 1), EXS + suf[0]) + for suf in cls._loader_handlers]) + suffixes.sort() + for neglen, suffix in suffixes: + if filename[neglen:] == suffix: + return filename[:neglen] + return None + + def iter_modules(self, prefix=''): + if self.path is None or not os.path.isdir(self.path): + return + + yielded = {} + + try: + filenames = os.listdir(self.path) + except OSError: + # ignore unreadable directories like import does + filenames = [] + filenames.sort() + for fn in filenames: + modname = self.getmodulename(fn) + if modname == '__init__' or modname in yielded: + continue + + path = os.path.join(self.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + try: + dircontents = os.listdir(path) + except OSError: + # ignore unreadable directories like import does + dircontents = [] + for fn in dircontents: + subname = self.getmodulename(fn) + if subname == '__init__': + ispkg = True + break + else: + continue # not a package + + if modname and '.' not in modname: + yielded[modname] = 1 + yield prefix + modname, ispkg + + +def _polyloader_pathhook(path): + if not os.path.isdir(path): + raise ImportError('Only directories are supported: %s' % path) + return PolyFinder(path) + + +def install(compiler, suffixes): + if not PolyFinder._installed: + sys.path_hooks.append(_polyloader_pathhook) + PolyFinder._installed = True + PolyFinder._install(compiler, suffixes) + + +def reset(): + PolyFinder._loader_handlers = [] + PolyFinder._installed = False diff --git a/hy/importer/polyloader/_python3.py b/hy/importer/polyloader/_python3.py new file mode 100644 index 0000000..dbbbdb1 --- /dev/null +++ b/hy/importer/polyloader/_python3.py @@ -0,0 +1,296 @@ +import os +import sys +import marshal +import pkgutil +import types +import _imp + + +# Python 3 refactored importlib several times, resulting in +# critical pieces of infrastructure moving around or being +# renamed. All the NOQA here is to help flake8 get past the +# "redefinition" complaints. + +if sys.version_info[0:2] in [(3, 3), (3, 4)]: + from importlib._bootstrap import ( # NOQA + cache_from_source, SourceFileLoader, # NOQA + FileFinder, _verbose_message, # NOQA + _get_supported_file_loaders, _relax_case, # NOQA + _w_long, _code_type) # NOQA + + +if sys.version_info[0:2] in [(3, 3)]: + from importlib._bootstrap import _MAGIC_BYTES as MAGIC_NUMBER # NOQA + +if sys.version_info[0:2] == (3, 4): + from importlib._bootstrap import _validate_bytecode_header, MAGIC_NUMBER # NOQA + +if sys.version_info[0:2] >= (3, 5): + from importlib.machinery import SourceFileLoader, FileFinder # NOQA + from importlib._bootstrap import _verbose_message # NOQA + from importlib._bootstrap_external import ( # NOQA + _w_long, _code_type, cache_from_source, # NOQA + _validate_bytecode_header, # NOQA + MAGIC_NUMBER, _relax_case, # NOQA + _get_supported_file_loaders) # NOQA + +SEP = os.sep +EXS = os.extsep +FLS = [('%s' + SEP + '__init__' + EXS + '%s', True), + ('%s' + EXS + '%s', False)] + + +def _suffixer(loaders): + return [(suffix, loader) + for (loader, suffixes) in loaders + for suffix in suffixes] + + +class _PolySourceFileLoader(SourceFileLoader): + _compiler = None + + def _poly_bytes_from_bytecode(self, fullname, data, path, st): + if hasattr(self, '_bytes_from_bytecode'): + return self._bytes_from_bytecode(fullname, data, + path, st) + self_module = sys.modules[__name__] + if hasattr(self_module, '_validate_bytecode_header'): + return _validate_bytecode_header(data, source_stats=st, + name=fullname, path=path) + raise ImportError("No bytecode handler found loading.") + + # All this just to change one line. + def get_code(self, fullname): + source_path = self.get_filename(fullname) + source_mtime = None + try: + bytecode_path = cache_from_source(source_path) + except NotImplementedError: + bytecode_path = None + else: + try: + st = self.path_stats(source_path) + except NotImplementedError: + pass + else: + source_mtime = int(st['mtime']) + try: + data = self.get_data(bytecode_path) + except IOError: + pass + else: + try: + bytes_data = self._poly_bytes_from_bytecode( + fullname, data, + bytecode_path, + st) + except (ImportError, EOFError): + pass + else: + _verbose_message( + '{} matches {}', + bytecode_path, + source_path) + found = marshal.loads(bytes_data) + if isinstance(found, _code_type): + _imp._fix_co_filename(found, source_path) + _verbose_message( + 'code object from {}', + bytecode_path) + return found + else: + msg = "Non-code object in {}" + raise ImportError( + msg.format(bytecode_path), + name=fullname, + path=bytecode_path) + source_bytes = self.get_data(source_path) + code_object = self._compiler(source_bytes, source_path, fullname) + _verbose_message('code object from {}', source_path) + if (not sys.dont_write_bytecode and + bytecode_path is not None and + source_mtime is not None): + data = bytearray(MAGIC_NUMBER) + data.extend(_w_long(source_mtime)) + data.extend(_w_long(len(source_bytes))) + data.extend(marshal.dumps(code_object)) + try: + self._cache_bytecode(source_path, bytecode_path, data) + _verbose_message('wrote {!r}', bytecode_path) + except NotImplementedError: + pass + return code_object + + +class PolyFileFinder(FileFinder): + '''The poly version of FileFinder supports the addition of loaders + after initialization. That's pretty much the whole point of the + PolyLoader mechanism.''' + + _native_loaders = [] + _custom_loaders = [] + + def __init__(self, path): + # Base (directory) path + self.path = path or '.' + self._path_mtime = -1 + self._path_cache = set() + self._relaxed_path_cache = set() + + @property + def _loaders(self): + return self._custom_loaders + list(self._native_loaders) + + @classmethod + def _install(cls, compiler, suffixes): + if not suffixes: + return + if isinstance(suffixes, str): + suffixes = [suffixes] + suffixset = set(suffixes) + overlap = suffixset.intersection( + set([suf[0] for suf in cls._native_loaders])) + if overlap: + r = "Override of native Python extensions is not permitted." + raise RuntimeError(r) + overlap = suffixset.intersection( + set([loader[0] for loader in cls._custom_loaders])) + if overlap: + # Fail silently + return + + newloaderclassname = ( + suffixes[0].lower().capitalize() + + str(_PolySourceFileLoader).rpartition('.')[2][1:]) + if isinstance(compiler, types.FunctionType): + newloader = type(newloaderclassname, (_PolySourceFileLoader,), + dict(_compiler=staticmethod(compiler))) + else: + newloader = type(newloaderclassname, (_PolySourceFileLoader,), + dict(_compiler=compiler)) + cls._custom_loaders += [(EXS + suffix, newloader) + for suffix in suffixset] + + @classmethod + def getmodulename(cls, path): + filename = os.path.basename(path) + suffixes = ([(-len(suf[0]), suf[0]) for suf in cls._native_loaders] + + [(-len(suf[0]), suf[0]) for suf in cls._custom_loaders]) + suffixes.sort() + for neglen, suffix in suffixes: + if filename[neglen:] == suffix: + return filename[:neglen] + return None + + def find_loader(self, fullname): + """Try to find a loader for the specified module, or the namespace + package portions. Returns (loader, list-of-portions).""" + is_namespace = False + tail_module = fullname.rpartition('.')[2] + try: + mtime = os.stat(self.path).st_mtime + except OSError: + mtime = -1 + if mtime != self._path_mtime: + self._fill_cache() + self._path_mtime = mtime + # tail_module keeps the original casing, for __file__ and friends + if _relax_case(): + cache = self._relaxed_path_cache + cache_module = tail_module.lower() + else: + cache = self._path_cache + cache_module = tail_module + # Check if the module is the name of a directory (and thus a package). + if cache_module in cache: + base_path = os.path.join(self.path, tail_module) + if os.path.isdir(base_path): + for suffix, loader in self._loaders: + init_filename = '__init__' + suffix + full_path = os.path.join(base_path, init_filename) + if os.path.isfile(full_path): + return (loader(fullname, full_path), [base_path]) + else: + # A namespace package, return the path if we don't also + # find a module in the next section. + is_namespace = True + # Check for a file w/ a proper suffix exists. + for suffix, loader in self._loaders: + full_path = os.path.join(self.path, tail_module + suffix) + _verbose_message('trying {}'.format(full_path), verbosity=2) + if cache_module + suffix in cache: + if os.path.isfile(full_path): + return (loader(fullname, full_path), []) + if is_namespace: + _verbose_message('possible namespace for {}'.format(base_path)) + return (None, [base_path]) + return (None, []) + + @classmethod + def path_hook(cls, *loader_details): + cls._native_loaders = loader_details + + def path_hook_for_PolyFileFinder(path): + if not os.path.isdir(path): + raise ImportError("only directories are supported", path=path) + return PolyFileFinder(path) + return path_hook_for_PolyFileFinder + + +def _poly_file_finder_modules(importer, prefix=''): + if importer.path is None or not os.path.isdir(importer.path): + return + + yielded = {} + + try: + filenames = os.listdir(importer.path) + except OSError: + # ignore unreadable directories like import does + filenames = [] + filenames.sort() + for fn in filenames: + modname = importer.getmodulename(fn) + if modname == '__init__' or modname in yielded: + continue + + path = os.path.join(importer.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + try: + dircontents = os.listdir(path) + except OSError: + # ignore unreadable directories like import does + dircontents = [] + for fn in dircontents: + subname = importer.getmodulename(fn) + if subname == '__init__': + ispkg = True + break + else: + continue # not a package + + if modname and '.' not in modname: + yielded[modname] = 1 + yield prefix + modname, ispkg + + +def install(compiler, suffixes): + filefinder = [(f, i) for i, f in enumerate(sys.path_hooks) + if repr(f).find('.path_hook_for_FileFinder') != -1] + if filefinder: + filefinder, fpos = filefinder[0] + sys.path_hooks[fpos] = PolyFileFinder.path_hook( + *(_suffixer(_get_supported_file_loaders()))) + sys.path_importer_cache = {} + pkgutil.iter_importer_modules.register( + PolyFileFinder, + _poly_file_finder_modules) + + PolyFileFinder._install(compiler, suffixes) + + +def reset(): + PolyFileFinder._custom_loaders = [] diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index edfbb5a..6ff8a4e 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -1,6 +1,5 @@ -from hy.importer import import_file_to_module, import_buffer_to_ast, MetaLoader +from hy.importer import import_file_to_module, import_buffer_to_ast from hy.errors import HyTypeError -import os import ast @@ -17,12 +16,9 @@ def test_stringer(): def test_imports(): - path = os.getcwd() + "/tests/resources/importer/a.hy" - testLoader = MetaLoader(path) - def _import_test(): try: - return testLoader.load_module("tests.resources.importer.a") + import tests.resources.importer.a # NOQA except: return "Error"