diff --git a/.gitignore b/.gitignore index d3bb39c..f1cb8d0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,3 @@ dist .coverage build/ .noseids -*~ -\#* -.\#* diff --git a/hy/cmdline.py b/hy/cmdline.py index 8bc13f4..b938df2 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -196,13 +196,11 @@ def run_command(source): def run_module(mod_name): - import pkgutil - mod = next((mod for mod in pkgutil.walk_packages() - if mod[1] == mod_name), None) - if mod is not None: - loader = mod[0].find_module(mod_name) - sys.argv = [loader.path] + sys.argv - return run_file(loader.path) + from hy.importer import MetaImporter + pth = MetaImporter().find_on_path(mod_name) + if pth is not None: + sys.argv = [pth] + sys.argv + return run_file(pth) print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name), file=sys.stderr) diff --git a/hy/importer/__init__.py b/hy/importer.py similarity index 76% rename from hy/importer/__init__.py rename to hy/importer.py index 826497e..8418b7c 100644 --- a/hy/importer/__init__.py +++ b/hy/importer.py @@ -23,7 +23,6 @@ from hy.compiler import hy_compile, HyTypeError from hy.models import HyObject, replace_hy_obj from hy.lex import tokenize, LexException from hy.errors import HyIOError -from hy.importer import polyloader from io import open import marshal @@ -173,23 +172,63 @@ def write_hy_as_pyc(fname): fc.write(MAGIC) -def _compile_hy(source_text, filename, fullname, *extra): - try: - flags = (__future__.CO_FUTURE_DIVISION | - __future__.CO_FUTURE_PRINT_FUNCTION) - return compile( - hy_compile( - import_buffer_to_hst(source_text.decode('utf-8')), fullname), - filename, "exec", flags) - except (HyTypeError, LexException) as e: - if e.source is None: - with open(filename, 'rt') as fp: - e.source = fp.read() - e.filename = filename - raise - except Exception: - raise +class MetaLoader(object): + def __init__(self, path): + self.path = path -polyloader.install(_compile_hy, ['hy']) -if '' not in sys.path: - sys.path.insert(0, '') + def is_package(self, fullname): + dirpath = "/".join(fullname.split(".")) + for pth in sys.path: + pth = os.path.abspath(pth) + composed_path = "%s/%s/__init__.hy" % (pth, dirpath) + if os.path.exists(composed_path): + return True + return False + + def load_module(self, fullname): + if fullname in sys.modules: + return sys.modules[fullname] + + if not self.path: + return + + sys.modules[fullname] = None + mod = import_file_to_module(fullname, + self.path) + + ispkg = self.is_package(fullname) + + mod.__file__ = self.path + mod.__loader__ = self + mod.__name__ = fullname + + if ispkg: + mod.__path__ = [] + mod.__package__ = fullname + else: + mod.__package__ = fullname.rpartition('.')[0] + + sys.modules[fullname] = mod + return mod + + +class MetaImporter(object): + def find_on_path(self, fullname): + fls = ["%s/__init__.hy", "%s.hy"] + dirpath = "/".join(fullname.split(".")) + + for pth in sys.path: + pth = os.path.abspath(pth) + for fp in fls: + composed_path = fp % ("%s/%s" % (pth, dirpath)) + if os.path.exists(composed_path): + return composed_path + + def find_module(self, fullname, path=None): + path = self.find_on_path(fullname) + if path: + return MetaLoader(path) + + +sys.meta_path.insert(0, MetaImporter()) +sys.path.insert(0, "") diff --git a/hy/importer/polyloader/__init__.py b/hy/importer/polyloader/__init__.py deleted file mode 100644 index 26b20c5..0000000 --- a/hy/importer/polyloader/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -import sys - -__author__ = 'Kenneth M. "Elf" Sternberg' -__email__ = 'elf.sternberg@gmail.com' -__version__ = '0.1.0' - -if sys.version_info[0:2] >= (2, 6): - from ._python2 import install, reset # NOQA - -if sys.version_info[0] >= 3: - from ._python3 import install, reset # NOQA - -__all__ = ['install', 'reset'] diff --git a/hy/importer/polyloader/_python2.py b/hy/importer/polyloader/_python2.py deleted file mode 100644 index dcc1cd8..0000000 --- a/hy/importer/polyloader/_python2.py +++ /dev/null @@ -1,188 +0,0 @@ -import io -import os -import os.path -import sys -import imp -import types -import pkgutil -from collections import namedtuple - - -SEP = os.sep -EXS = os.extsep -FLS = [('%s' + SEP + '__init__' + EXS + '%s', True), - ('%s' + EXS + '%s', False)] - -Loader = namedtuple('Loader', 'suffix compiler') - - -class PolyLoader(): - def __init__(self, fullname, path, is_pkg): - self.fullname = fullname - self.path = path - self.is_package = is_pkg - - def load_module(self, fullname): - if fullname in sys.modules: - return sys.modules[fullname] - - if fullname != self.fullname: - raise ImportError("Load confusion: %s vs %s." % - (fullname, self.fullname)) - - matches = [loader for loader in PolyFinder._loader_handlers - if self.path.endswith(loader.suffix)] - - if len(matches) == 0: - raise ImportError("%s is not a recognized module?" % fullname) - - if len(matches) > 1: - raise ImportError("Multiple possible resolutions for %s: %s" % ( - fullname, ', '.join([loader.suffix for loader in matches]))) - - compiler = matches[0].compiler - with io.FileIO(self.path, 'r') as file: - source_text = file.read() - - code = compiler(source_text, self.path, fullname) - - module = types.ModuleType(fullname) - module.__file__ = self.path - module.__name__ = fullname - module.__package__ = '.'.join(fullname.split('.')[:-1]) - - if self.is_package: - module.__path__ = [os.path.dirname(module.__file__)] - module.__package__ = fullname - - exec(code, module.__dict__) - sys.modules[fullname] = module - return module - - -# PolyFinder is an implementation of the Finder class from Python 2.7, -# with embellishments gleefully copied from Python 3.4. It supports -# all the same functionality for non-.py sourcefiles with the added -# benefit of falling back to Python's default behavior. - -# Polyfinder is instantiated by _polyloader_pathhook() - -class PolyFinder(object): - _loader_handlers = [] - _installed = False - - def __init__(self, path=None): - self.path = path or '.' - - def _pl_find_on_path(self, fullname, path=None): - subname = fullname.split(".")[-1] - if self.path is None and subname != fullname: - return None - - path = os.path.realpath(self.path) - for (fp, ispkg) in FLS: - for loader in self._loader_handlers: - composed_path = fp % (('%s' + SEP + '%s') % - (path, subname), loader.suffix) - if os.path.isdir(composed_path): - r = "Invalid: Directory name ends in recognized suffix" - raise IOError(r) - if os.path.isfile(composed_path): - return PolyLoader(fullname, composed_path, ispkg) - - # Fall back onto Python's own methods. - try: - file, filename, etc = imp.find_module(subname, [path]) - except ImportError as e: # NOQA - return None - return pkgutil.ImpLoader(fullname, file, filename, etc) - - def find_module(self, fullname, path=None): - return self._pl_find_on_path(fullname) - - @classmethod - def _install(cls, compiler, suffixes): - if isinstance(suffixes, basestring): - suffixes = [suffixes] - suffixes = set(suffixes) - overlap = suffixes.intersection( - set([suf[0] for suf in imp.get_suffixes()])) - if overlap: - r = "Override of native Python extensions is not permitted." - raise RuntimeError(r) - overlap = suffixes.intersection( - set([loader.suffix for loader in cls._loader_handlers])) - if overlap: - # Fail silently - return - cls._loader_handlers += [Loader(suf, compiler) for suf in suffixes] - - @classmethod - def getmodulename(cls, path): - filename = os.path.basename(path) - suffixes = ([(-len(suf[0]), suf[0]) - for suf in imp.get_suffixes()] + - [(-(len(suf[0]) + 1), EXS + suf[0]) - for suf in cls._loader_handlers]) - suffixes.sort() - for neglen, suffix in suffixes: - if filename[neglen:] == suffix: - return filename[:neglen] - return None - - def iter_modules(self, prefix=''): - if self.path is None or not os.path.isdir(self.path): - return - - yielded = {} - - try: - filenames = os.listdir(self.path) - except OSError: - # ignore unreadable directories like import does - filenames = [] - filenames.sort() - for fn in filenames: - modname = self.getmodulename(fn) - if modname == '__init__' or modname in yielded: - continue - - path = os.path.join(self.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - try: - dircontents = os.listdir(path) - except OSError: - # ignore unreadable directories like import does - dircontents = [] - for fn in dircontents: - subname = self.getmodulename(fn) - if subname == '__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - - -def _polyloader_pathhook(path): - if not os.path.isdir(path): - raise ImportError('Only directories are supported: %s' % path) - return PolyFinder(path) - - -def install(compiler, suffixes): - if not PolyFinder._installed: - sys.path_hooks.append(_polyloader_pathhook) - PolyFinder._installed = True - PolyFinder._install(compiler, suffixes) - - -def reset(): - PolyFinder._loader_handlers = [] - PolyFinder._installed = False diff --git a/hy/importer/polyloader/_python3.py b/hy/importer/polyloader/_python3.py deleted file mode 100644 index dbbbdb1..0000000 --- a/hy/importer/polyloader/_python3.py +++ /dev/null @@ -1,296 +0,0 @@ -import os -import sys -import marshal -import pkgutil -import types -import _imp - - -# Python 3 refactored importlib several times, resulting in -# critical pieces of infrastructure moving around or being -# renamed. All the NOQA here is to help flake8 get past the -# "redefinition" complaints. - -if sys.version_info[0:2] in [(3, 3), (3, 4)]: - from importlib._bootstrap import ( # NOQA - cache_from_source, SourceFileLoader, # NOQA - FileFinder, _verbose_message, # NOQA - _get_supported_file_loaders, _relax_case, # NOQA - _w_long, _code_type) # NOQA - - -if sys.version_info[0:2] in [(3, 3)]: - from importlib._bootstrap import _MAGIC_BYTES as MAGIC_NUMBER # NOQA - -if sys.version_info[0:2] == (3, 4): - from importlib._bootstrap import _validate_bytecode_header, MAGIC_NUMBER # NOQA - -if sys.version_info[0:2] >= (3, 5): - from importlib.machinery import SourceFileLoader, FileFinder # NOQA - from importlib._bootstrap import _verbose_message # NOQA - from importlib._bootstrap_external import ( # NOQA - _w_long, _code_type, cache_from_source, # NOQA - _validate_bytecode_header, # NOQA - MAGIC_NUMBER, _relax_case, # NOQA - _get_supported_file_loaders) # NOQA - -SEP = os.sep -EXS = os.extsep -FLS = [('%s' + SEP + '__init__' + EXS + '%s', True), - ('%s' + EXS + '%s', False)] - - -def _suffixer(loaders): - return [(suffix, loader) - for (loader, suffixes) in loaders - for suffix in suffixes] - - -class _PolySourceFileLoader(SourceFileLoader): - _compiler = None - - def _poly_bytes_from_bytecode(self, fullname, data, path, st): - if hasattr(self, '_bytes_from_bytecode'): - return self._bytes_from_bytecode(fullname, data, - path, st) - self_module = sys.modules[__name__] - if hasattr(self_module, '_validate_bytecode_header'): - return _validate_bytecode_header(data, source_stats=st, - name=fullname, path=path) - raise ImportError("No bytecode handler found loading.") - - # All this just to change one line. - def get_code(self, fullname): - source_path = self.get_filename(fullname) - source_mtime = None - try: - bytecode_path = cache_from_source(source_path) - except NotImplementedError: - bytecode_path = None - else: - try: - st = self.path_stats(source_path) - except NotImplementedError: - pass - else: - source_mtime = int(st['mtime']) - try: - data = self.get_data(bytecode_path) - except IOError: - pass - else: - try: - bytes_data = self._poly_bytes_from_bytecode( - fullname, data, - bytecode_path, - st) - except (ImportError, EOFError): - pass - else: - _verbose_message( - '{} matches {}', - bytecode_path, - source_path) - found = marshal.loads(bytes_data) - if isinstance(found, _code_type): - _imp._fix_co_filename(found, source_path) - _verbose_message( - 'code object from {}', - bytecode_path) - return found - else: - msg = "Non-code object in {}" - raise ImportError( - msg.format(bytecode_path), - name=fullname, - path=bytecode_path) - source_bytes = self.get_data(source_path) - code_object = self._compiler(source_bytes, source_path, fullname) - _verbose_message('code object from {}', source_path) - if (not sys.dont_write_bytecode and - bytecode_path is not None and - source_mtime is not None): - data = bytearray(MAGIC_NUMBER) - data.extend(_w_long(source_mtime)) - data.extend(_w_long(len(source_bytes))) - data.extend(marshal.dumps(code_object)) - try: - self._cache_bytecode(source_path, bytecode_path, data) - _verbose_message('wrote {!r}', bytecode_path) - except NotImplementedError: - pass - return code_object - - -class PolyFileFinder(FileFinder): - '''The poly version of FileFinder supports the addition of loaders - after initialization. That's pretty much the whole point of the - PolyLoader mechanism.''' - - _native_loaders = [] - _custom_loaders = [] - - def __init__(self, path): - # Base (directory) path - self.path = path or '.' - self._path_mtime = -1 - self._path_cache = set() - self._relaxed_path_cache = set() - - @property - def _loaders(self): - return self._custom_loaders + list(self._native_loaders) - - @classmethod - def _install(cls, compiler, suffixes): - if not suffixes: - return - if isinstance(suffixes, str): - suffixes = [suffixes] - suffixset = set(suffixes) - overlap = suffixset.intersection( - set([suf[0] for suf in cls._native_loaders])) - if overlap: - r = "Override of native Python extensions is not permitted." - raise RuntimeError(r) - overlap = suffixset.intersection( - set([loader[0] for loader in cls._custom_loaders])) - if overlap: - # Fail silently - return - - newloaderclassname = ( - suffixes[0].lower().capitalize() + - str(_PolySourceFileLoader).rpartition('.')[2][1:]) - if isinstance(compiler, types.FunctionType): - newloader = type(newloaderclassname, (_PolySourceFileLoader,), - dict(_compiler=staticmethod(compiler))) - else: - newloader = type(newloaderclassname, (_PolySourceFileLoader,), - dict(_compiler=compiler)) - cls._custom_loaders += [(EXS + suffix, newloader) - for suffix in suffixset] - - @classmethod - def getmodulename(cls, path): - filename = os.path.basename(path) - suffixes = ([(-len(suf[0]), suf[0]) for suf in cls._native_loaders] + - [(-len(suf[0]), suf[0]) for suf in cls._custom_loaders]) - suffixes.sort() - for neglen, suffix in suffixes: - if filename[neglen:] == suffix: - return filename[:neglen] - return None - - def find_loader(self, fullname): - """Try to find a loader for the specified module, or the namespace - package portions. Returns (loader, list-of-portions).""" - is_namespace = False - tail_module = fullname.rpartition('.')[2] - try: - mtime = os.stat(self.path).st_mtime - except OSError: - mtime = -1 - if mtime != self._path_mtime: - self._fill_cache() - self._path_mtime = mtime - # tail_module keeps the original casing, for __file__ and friends - if _relax_case(): - cache = self._relaxed_path_cache - cache_module = tail_module.lower() - else: - cache = self._path_cache - cache_module = tail_module - # Check if the module is the name of a directory (and thus a package). - if cache_module in cache: - base_path = os.path.join(self.path, tail_module) - if os.path.isdir(base_path): - for suffix, loader in self._loaders: - init_filename = '__init__' + suffix - full_path = os.path.join(base_path, init_filename) - if os.path.isfile(full_path): - return (loader(fullname, full_path), [base_path]) - else: - # A namespace package, return the path if we don't also - # find a module in the next section. - is_namespace = True - # Check for a file w/ a proper suffix exists. - for suffix, loader in self._loaders: - full_path = os.path.join(self.path, tail_module + suffix) - _verbose_message('trying {}'.format(full_path), verbosity=2) - if cache_module + suffix in cache: - if os.path.isfile(full_path): - return (loader(fullname, full_path), []) - if is_namespace: - _verbose_message('possible namespace for {}'.format(base_path)) - return (None, [base_path]) - return (None, []) - - @classmethod - def path_hook(cls, *loader_details): - cls._native_loaders = loader_details - - def path_hook_for_PolyFileFinder(path): - if not os.path.isdir(path): - raise ImportError("only directories are supported", path=path) - return PolyFileFinder(path) - return path_hook_for_PolyFileFinder - - -def _poly_file_finder_modules(importer, prefix=''): - if importer.path is None or not os.path.isdir(importer.path): - return - - yielded = {} - - try: - filenames = os.listdir(importer.path) - except OSError: - # ignore unreadable directories like import does - filenames = [] - filenames.sort() - for fn in filenames: - modname = importer.getmodulename(fn) - if modname == '__init__' or modname in yielded: - continue - - path = os.path.join(importer.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - try: - dircontents = os.listdir(path) - except OSError: - # ignore unreadable directories like import does - dircontents = [] - for fn in dircontents: - subname = importer.getmodulename(fn) - if subname == '__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - - -def install(compiler, suffixes): - filefinder = [(f, i) for i, f in enumerate(sys.path_hooks) - if repr(f).find('.path_hook_for_FileFinder') != -1] - if filefinder: - filefinder, fpos = filefinder[0] - sys.path_hooks[fpos] = PolyFileFinder.path_hook( - *(_suffixer(_get_supported_file_loaders()))) - sys.path_importer_cache = {} - pkgutil.iter_importer_modules.register( - PolyFileFinder, - _poly_file_finder_modules) - - PolyFileFinder._install(compiler, suffixes) - - -def reset(): - PolyFileFinder._custom_loaders = [] diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 6ff8a4e..edfbb5a 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -1,5 +1,6 @@ -from hy.importer import import_file_to_module, import_buffer_to_ast +from hy.importer import import_file_to_module, import_buffer_to_ast, MetaLoader from hy.errors import HyTypeError +import os import ast @@ -16,9 +17,12 @@ def test_stringer(): def test_imports(): + path = os.getcwd() + "/tests/resources/importer/a.hy" + testLoader = MetaLoader(path) + def _import_test(): try: - import tests.resources.importer.a # NOQA + return testLoader.load_module("tests.resources.importer.a") except: return "Error"