Revert "Modernized." (#1085) (#1188)

This reverts commit 13b484ce46 and fixes #1183.

.gitignore

@@ -9,6 +9,3 @@ dist
 .coverage
 build/
 .noseids
-*~
-\#*
-.\#*

hy/cmdline.py

@@ -196,13 +196,11 @@ def run_command(source):
 
 
 def run_module(mod_name):
-    import pkgutil
+    from hy.importer import MetaImporter
-    mod = next((mod for mod in pkgutil.walk_packages()
+    pth = MetaImporter().find_on_path(mod_name)
-                if mod[1] == mod_name), None)
+    if pth is not None:
-    if mod is not None:
+        sys.argv = [pth] + sys.argv
-        loader = mod[0].find_module(mod_name)
+        return run_file(pth)
-        sys.argv = [loader.path] + sys.argv
-        return run_file(loader.path)
 
     print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name),
           file=sys.stderr)

hy/importer.py

@@ -23,7 +23,6 @@ from hy.compiler import hy_compile, HyTypeError
 from hy.models import HyObject, replace_hy_obj
 from hy.lex import tokenize, LexException
 from hy.errors import HyIOError
-from hy.importer import polyloader
 
 from io import open
 import marshal
@@ -173,23 +172,63 @@ def write_hy_as_pyc(fname):
         fc.write(MAGIC)
 
 
-def _compile_hy(source_text, filename, fullname, *extra):
+class MetaLoader(object):
-    try:
+    def __init__(self, path):
-        flags = (__future__.CO_FUTURE_DIVISION |
+        self.path = path
-                 __future__.CO_FUTURE_PRINT_FUNCTION)
-        return compile(
-            hy_compile(
-                import_buffer_to_hst(source_text.decode('utf-8')), fullname),
-            filename, "exec", flags)
-    except (HyTypeError, LexException) as e:
-        if e.source is None:
-            with open(filename, 'rt') as fp:
-                e.source = fp.read()
-            e.filename = filename
-        raise
-    except Exception:
-        raise
 
-polyloader.install(_compile_hy, ['hy'])
+    def is_package(self, fullname):
-if '' not in sys.path:
+        dirpath = "/".join(fullname.split("."))
-    sys.path.insert(0, '')
+        for pth in sys.path:
+            pth = os.path.abspath(pth)
+            composed_path = "%s/%s/__init__.hy" % (pth, dirpath)
+            if os.path.exists(composed_path):
+                return True
+        return False
+
+    def load_module(self, fullname):
+        if fullname in sys.modules:
+            return sys.modules[fullname]
+
+        if not self.path:
+            return
+
+        sys.modules[fullname] = None
+        mod = import_file_to_module(fullname,
+                                    self.path)
+
+        ispkg = self.is_package(fullname)
+
+        mod.__file__ = self.path
+        mod.__loader__ = self
+        mod.__name__ = fullname
+
+        if ispkg:
+            mod.__path__ = []
+            mod.__package__ = fullname
+        else:
+            mod.__package__ = fullname.rpartition('.')[0]
+
+        sys.modules[fullname] = mod
+        return mod
+
+
+class MetaImporter(object):
+    def find_on_path(self, fullname):
+        fls = ["%s/__init__.hy", "%s.hy"]
+        dirpath = "/".join(fullname.split("."))
+
+        for pth in sys.path:
+            pth = os.path.abspath(pth)
+            for fp in fls:
+                composed_path = fp % ("%s/%s" % (pth, dirpath))
+                if os.path.exists(composed_path):
+                    return composed_path
+
+    def find_module(self, fullname, path=None):
+        path = self.find_on_path(fullname)
+        if path:
+            return MetaLoader(path)
+
+
+sys.meta_path.insert(0, MetaImporter())
+sys.path.insert(0, "")

hy/importer/polyloader/__init__.py

@@ -1,14 +0,0 @@
-# -*- coding: utf-8 -*-
-import sys
-
-__author__ = 'Kenneth M. "Elf" Sternberg'
-__email__ = 'elf.sternberg@gmail.com'
-__version__ = '0.1.0'
-
-if sys.version_info[0:2] >= (2, 6):
-    from ._python2 import install, reset  # NOQA
-
-if sys.version_info[0] >= 3:
-    from ._python3 import install, reset  # NOQA
-
-__all__ = ['install', 'reset']

hy/importer/polyloader/_python2.py

@@ -1,188 +0,0 @@
-import io
-import os
-import os.path
-import sys
-import imp
-import types
-import pkgutil
-from collections import namedtuple
-
-
-SEP = os.sep
-EXS = os.extsep
-FLS = [('%s' + SEP + '__init__' + EXS + '%s', True),
-       ('%s' + EXS + '%s', False)]
-
-Loader = namedtuple('Loader', 'suffix compiler')
-
-
-class PolyLoader():
-    def __init__(self, fullname, path, is_pkg):
-        self.fullname = fullname
-        self.path = path
-        self.is_package = is_pkg
-
-    def load_module(self, fullname):
-        if fullname in sys.modules:
-            return sys.modules[fullname]
-
-        if fullname != self.fullname:
-            raise ImportError("Load confusion: %s vs %s." %
-                              (fullname, self.fullname))
-
-        matches = [loader for loader in PolyFinder._loader_handlers
-                   if self.path.endswith(loader.suffix)]
-
-        if len(matches) == 0:
-            raise ImportError("%s is not a recognized module?" % fullname)
-
-        if len(matches) > 1:
-            raise ImportError("Multiple possible resolutions for %s: %s" % (
-                fullname, ', '.join([loader.suffix for loader in matches])))
-
-        compiler = matches[0].compiler
-        with io.FileIO(self.path, 'r') as file:
-            source_text = file.read()
-
-        code = compiler(source_text, self.path, fullname)
-
-        module = types.ModuleType(fullname)
-        module.__file__ = self.path
-        module.__name__ = fullname
-        module.__package__ = '.'.join(fullname.split('.')[:-1])
-
-        if self.is_package:
-            module.__path__ = [os.path.dirname(module.__file__)]
-            module.__package__ = fullname
-
-        exec(code, module.__dict__)
-        sys.modules[fullname] = module
-        return module
-
-
-# PolyFinder is an implementation of the Finder class from Python 2.7,
-# with embellishments gleefully copied from Python 3.4.  It supports
-# all the same functionality for non-.py sourcefiles with the added
-# benefit of falling back to Python's default behavior.
-
-# Polyfinder is instantiated by _polyloader_pathhook()
-
-class PolyFinder(object):
-    _loader_handlers = []
-    _installed = False
-
-    def __init__(self, path=None):
-        self.path = path or '.'
-
-    def _pl_find_on_path(self, fullname, path=None):
-        subname = fullname.split(".")[-1]
-        if self.path is None and subname != fullname:
-            return None
-
-        path = os.path.realpath(self.path)
-        for (fp, ispkg) in FLS:
-            for loader in self._loader_handlers:
-                composed_path = fp % (('%s' + SEP + '%s') %
-                                      (path, subname), loader.suffix)
-                if os.path.isdir(composed_path):
-                    r = "Invalid: Directory name ends in recognized suffix"
-                    raise IOError(r)
-                if os.path.isfile(composed_path):
-                    return PolyLoader(fullname, composed_path, ispkg)
-
-        # Fall back onto Python's own methods.
-        try:
-            file, filename, etc = imp.find_module(subname, [path])
-        except ImportError as e:  # NOQA
-            return None
-        return pkgutil.ImpLoader(fullname, file, filename, etc)
-
-    def find_module(self, fullname, path=None):
-        return self._pl_find_on_path(fullname)
-
-    @classmethod
-    def _install(cls, compiler, suffixes):
-        if isinstance(suffixes, basestring):
-            suffixes = [suffixes]
-        suffixes = set(suffixes)
-        overlap = suffixes.intersection(
-            set([suf[0] for suf in imp.get_suffixes()]))
-        if overlap:
-            r = "Override of native Python extensions is not permitted."
-            raise RuntimeError(r)
-        overlap = suffixes.intersection(
-            set([loader.suffix for loader in cls._loader_handlers]))
-        if overlap:
-            # Fail silently
-            return
-        cls._loader_handlers += [Loader(suf, compiler) for suf in suffixes]
-
-    @classmethod
-    def getmodulename(cls, path):
-        filename = os.path.basename(path)
-        suffixes = ([(-len(suf[0]), suf[0])
-                     for suf in imp.get_suffixes()] +
-                    [(-(len(suf[0]) + 1), EXS + suf[0])
-                     for suf in cls._loader_handlers])
-        suffixes.sort()
-        for neglen, suffix in suffixes:
-            if filename[neglen:] == suffix:
-                return filename[:neglen]
-        return None
-
-    def iter_modules(self, prefix=''):
-        if self.path is None or not os.path.isdir(self.path):
-            return
-
-        yielded = {}
-
-        try:
-            filenames = os.listdir(self.path)
-        except OSError:
-            # ignore unreadable directories like import does
-            filenames = []
-        filenames.sort()
-        for fn in filenames:
-            modname = self.getmodulename(fn)
-            if modname == '__init__' or modname in yielded:
-                continue
-
-            path = os.path.join(self.path, fn)
-            ispkg = False
-
-            if not modname and os.path.isdir(path) and '.' not in fn:
-                modname = fn
-                try:
-                    dircontents = os.listdir(path)
-                except OSError:
-                    # ignore unreadable directories like import does
-                    dircontents = []
-                for fn in dircontents:
-                    subname = self.getmodulename(fn)
-                    if subname == '__init__':
-                        ispkg = True
-                        break
-                else:
-                    continue    # not a package
-
-            if modname and '.' not in modname:
-                yielded[modname] = 1
-                yield prefix + modname, ispkg
-
-
-def _polyloader_pathhook(path):
-    if not os.path.isdir(path):
-        raise ImportError('Only directories are supported: %s' % path)
-    return PolyFinder(path)
-
-
-def install(compiler, suffixes):
-    if not PolyFinder._installed:
-        sys.path_hooks.append(_polyloader_pathhook)
-        PolyFinder._installed = True
-    PolyFinder._install(compiler, suffixes)
-
-
-def reset():
-    PolyFinder._loader_handlers = []
-    PolyFinder._installed = False

hy/importer/polyloader/_python3.py

@@ -1,296 +0,0 @@
-import os
-import sys
-import marshal
-import pkgutil
-import types
-import _imp
-
-
-# Python 3 refactored importlib several times, resulting in
-# critical pieces of infrastructure moving around or being
-# renamed.  All the NOQA here is to help flake8 get past the
-# "redefinition" complaints.
-
-if sys.version_info[0:2] in [(3, 3), (3, 4)]:
-    from importlib._bootstrap import (  # NOQA
-        cache_from_source, SourceFileLoader,  # NOQA
-        FileFinder, _verbose_message,  # NOQA
-        _get_supported_file_loaders, _relax_case,  # NOQA
-        _w_long, _code_type)  # NOQA
-
-
-if sys.version_info[0:2] in [(3, 3)]:
-    from importlib._bootstrap import _MAGIC_BYTES as MAGIC_NUMBER  # NOQA
-
-if sys.version_info[0:2] == (3, 4):
-    from importlib._bootstrap import _validate_bytecode_header, MAGIC_NUMBER  # NOQA
-
-if sys.version_info[0:2] >= (3, 5):
-    from importlib.machinery import SourceFileLoader, FileFinder  # NOQA
-    from importlib._bootstrap import _verbose_message  # NOQA
-    from importlib._bootstrap_external import (  # NOQA
-        _w_long, _code_type, cache_from_source,  # NOQA
-        _validate_bytecode_header,  # NOQA
-        MAGIC_NUMBER, _relax_case,  # NOQA
-        _get_supported_file_loaders)  # NOQA
-
-SEP = os.sep
-EXS = os.extsep
-FLS = [('%s' + SEP + '__init__' + EXS + '%s', True),
-       ('%s' + EXS + '%s', False)]
-
-
-def _suffixer(loaders):
-    return [(suffix, loader)
-            for (loader, suffixes) in loaders
-            for suffix in suffixes]
-
-
-class _PolySourceFileLoader(SourceFileLoader):
-    _compiler = None
-
-    def _poly_bytes_from_bytecode(self, fullname, data, path, st):
-        if hasattr(self, '_bytes_from_bytecode'):
-            return self._bytes_from_bytecode(fullname, data,
-                                             path, st)
-        self_module = sys.modules[__name__]
-        if hasattr(self_module, '_validate_bytecode_header'):
-            return _validate_bytecode_header(data, source_stats=st,
-                                             name=fullname, path=path)
-        raise ImportError("No bytecode handler found loading.")
-
-    # All this just to change one line.
-    def get_code(self, fullname):
-        source_path = self.get_filename(fullname)
-        source_mtime = None
-        try:
-            bytecode_path = cache_from_source(source_path)
-        except NotImplementedError:
-            bytecode_path = None
-        else:
-            try:
-                st = self.path_stats(source_path)
-            except NotImplementedError:
-                pass
-            else:
-                source_mtime = int(st['mtime'])
-                try:
-                    data = self.get_data(bytecode_path)
-                except IOError:
-                    pass
-                else:
-                    try:
-                        bytes_data = self._poly_bytes_from_bytecode(
-                            fullname, data,
-                            bytecode_path,
-                            st)
-                    except (ImportError, EOFError):
-                        pass
-                    else:
-                        _verbose_message(
-                            '{} matches {}',
-                            bytecode_path,
-                            source_path)
-                        found = marshal.loads(bytes_data)
-                        if isinstance(found, _code_type):
-                            _imp._fix_co_filename(found, source_path)
-                            _verbose_message(
-                                'code object from {}',
-                                bytecode_path)
-                            return found
-                        else:
-                            msg = "Non-code object in {}"
-                            raise ImportError(
-                                msg.format(bytecode_path),
-                                name=fullname,
-                                path=bytecode_path)
-        source_bytes = self.get_data(source_path)
-        code_object = self._compiler(source_bytes, source_path, fullname)
-        _verbose_message('code object from {}', source_path)
-        if (not sys.dont_write_bytecode and
-                bytecode_path is not None and
-                source_mtime is not None):
-            data = bytearray(MAGIC_NUMBER)
-            data.extend(_w_long(source_mtime))
-            data.extend(_w_long(len(source_bytes)))
-            data.extend(marshal.dumps(code_object))
-            try:
-                self._cache_bytecode(source_path, bytecode_path, data)
-                _verbose_message('wrote {!r}', bytecode_path)
-            except NotImplementedError:
-                pass
-        return code_object
-
-
-class PolyFileFinder(FileFinder):
-    '''The poly version of FileFinder supports the addition of loaders
-       after initialization.  That's pretty much the whole point of the
-       PolyLoader mechanism.'''
-
-    _native_loaders = []
-    _custom_loaders = []
-
-    def __init__(self, path):
-        # Base (directory) path
-        self.path = path or '.'
-        self._path_mtime = -1
-        self._path_cache = set()
-        self._relaxed_path_cache = set()
-
-    @property
-    def _loaders(self):
-        return self._custom_loaders + list(self._native_loaders)
-
-    @classmethod
-    def _install(cls, compiler, suffixes):
-        if not suffixes:
-            return
-        if isinstance(suffixes, str):
-            suffixes = [suffixes]
-        suffixset = set(suffixes)
-        overlap = suffixset.intersection(
-            set([suf[0] for suf in cls._native_loaders]))
-        if overlap:
-            r = "Override of native Python extensions is not permitted."
-            raise RuntimeError(r)
-        overlap = suffixset.intersection(
-            set([loader[0] for loader in cls._custom_loaders]))
-        if overlap:
-            # Fail silently
-            return
-
-        newloaderclassname = (
-            suffixes[0].lower().capitalize() +
-            str(_PolySourceFileLoader).rpartition('.')[2][1:])
-        if isinstance(compiler, types.FunctionType):
-            newloader = type(newloaderclassname, (_PolySourceFileLoader,),
-                             dict(_compiler=staticmethod(compiler)))
-        else:
-            newloader = type(newloaderclassname, (_PolySourceFileLoader,),
-                             dict(_compiler=compiler))
-        cls._custom_loaders += [(EXS + suffix, newloader)
-                                for suffix in suffixset]
-
-    @classmethod
-    def getmodulename(cls, path):
-        filename = os.path.basename(path)
-        suffixes = ([(-len(suf[0]), suf[0]) for suf in cls._native_loaders] +
-                    [(-len(suf[0]), suf[0]) for suf in cls._custom_loaders])
-        suffixes.sort()
-        for neglen, suffix in suffixes:
-            if filename[neglen:] == suffix:
-                return filename[:neglen]
-        return None
-
-    def find_loader(self, fullname):
-        """Try to find a loader for the specified module, or the namespace
-        package portions. Returns (loader, list-of-portions)."""
-        is_namespace = False
-        tail_module = fullname.rpartition('.')[2]
-        try:
-            mtime = os.stat(self.path).st_mtime
-        except OSError:
-            mtime = -1
-        if mtime != self._path_mtime:
-            self._fill_cache()
-            self._path_mtime = mtime
-        # tail_module keeps the original casing, for __file__ and friends
-        if _relax_case():
-            cache = self._relaxed_path_cache
-            cache_module = tail_module.lower()
-        else:
-            cache = self._path_cache
-            cache_module = tail_module
-        # Check if the module is the name of a directory (and thus a package).
-        if cache_module in cache:
-            base_path = os.path.join(self.path, tail_module)
-            if os.path.isdir(base_path):
-                for suffix, loader in self._loaders:
-                    init_filename = '__init__' + suffix
-                    full_path = os.path.join(base_path, init_filename)
-                    if os.path.isfile(full_path):
-                        return (loader(fullname, full_path), [base_path])
-                else:
-                    # A namespace package, return the path if we don't also
-                    #  find a module in the next section.
-                    is_namespace = True
-        # Check for a file w/ a proper suffix exists.
-        for suffix, loader in self._loaders:
-            full_path = os.path.join(self.path, tail_module + suffix)
-            _verbose_message('trying {}'.format(full_path), verbosity=2)
-            if cache_module + suffix in cache:
-                if os.path.isfile(full_path):
-                    return (loader(fullname, full_path), [])
-        if is_namespace:
-            _verbose_message('possible namespace for {}'.format(base_path))
-            return (None, [base_path])
-        return (None, [])
-
-    @classmethod
-    def path_hook(cls, *loader_details):
-        cls._native_loaders = loader_details
-
-        def path_hook_for_PolyFileFinder(path):
-            if not os.path.isdir(path):
-                raise ImportError("only directories are supported", path=path)
-            return PolyFileFinder(path)
-        return path_hook_for_PolyFileFinder
-
-
-def _poly_file_finder_modules(importer, prefix=''):
-    if importer.path is None or not os.path.isdir(importer.path):
-        return
-
-    yielded = {}
-
-    try:
-        filenames = os.listdir(importer.path)
-    except OSError:
-        # ignore unreadable directories like import does
-        filenames = []
-    filenames.sort()
-    for fn in filenames:
-        modname = importer.getmodulename(fn)
-        if modname == '__init__' or modname in yielded:
-            continue
-
-        path = os.path.join(importer.path, fn)
-        ispkg = False
-
-        if not modname and os.path.isdir(path) and '.' not in fn:
-            modname = fn
-            try:
-                dircontents = os.listdir(path)
-            except OSError:
-                # ignore unreadable directories like import does
-                dircontents = []
-            for fn in dircontents:
-                subname = importer.getmodulename(fn)
-                if subname == '__init__':
-                    ispkg = True
-                    break
-            else:
-                continue    # not a package
-
-        if modname and '.' not in modname:
-            yielded[modname] = 1
-            yield prefix + modname, ispkg
-
-
-def install(compiler, suffixes):
-    filefinder = [(f, i) for i, f in enumerate(sys.path_hooks)
-                  if repr(f).find('.path_hook_for_FileFinder') != -1]
-    if filefinder:
-        filefinder, fpos = filefinder[0]
-        sys.path_hooks[fpos] = PolyFileFinder.path_hook(
-            *(_suffixer(_get_supported_file_loaders())))
-        sys.path_importer_cache = {}
-        pkgutil.iter_importer_modules.register(
-            PolyFileFinder,
-            _poly_file_finder_modules)
-
-    PolyFileFinder._install(compiler, suffixes)
-
-
-def reset():
-    PolyFileFinder._custom_loaders = []

tests/importer/test_importer.py

@@ -1,5 +1,6 @@
-from hy.importer import import_file_to_module, import_buffer_to_ast
+from hy.importer import import_file_to_module, import_buffer_to_ast, MetaLoader
 from hy.errors import HyTypeError
+import os
 import ast
 
 
@@ -16,9 +17,12 @@ def test_stringer():
 
 
 def test_imports():
+    path = os.getcwd() + "/tests/resources/importer/a.hy"
+    testLoader = MetaLoader(path)
+
     def _import_test():
         try:
-            import tests.resources.importer.a  # NOQA
+            return testLoader.load_module("tests.resources.importer.a")
         except:
             return "Error"