Modernized.

.gitignore

@@ -9,3 +9,6 @@ dist
 .coverage
 build/
 .noseids
+*~
+\#*
+.\#*

hy/cmdline.py

@@ -196,11 +196,13 @@ def run_command(source):
 
 
 def run_module(mod_name):
-    from hy.importer import MetaImporter
+    import pkgutil
-    pth = MetaImporter().find_on_path(mod_name)
+    mod = next((mod for mod in pkgutil.walk_packages()
-    if pth is not None:
+                if mod[1] == mod_name), None)
-        sys.argv = [pth] + sys.argv
+    if mod is not None:
-        return run_file(pth)
+        loader = mod[0].find_module(mod_name)
+        sys.argv = [loader.path] + sys.argv
+        return run_file(loader.path)
 
     print("{0}: module '{1}' not found.\n".format(hy.__appname__, mod_name),
           file=sys.stderr)

hy/importer/__init__.py

@@ -23,6 +23,7 @@ from hy.compiler import hy_compile, HyTypeError
 from hy.models import HyObject, replace_hy_obj
 from hy.lex import tokenize, LexException
 from hy.errors import HyIOError
+from hy.importer import polyloader
 
 from io import open
 import marshal
@@ -172,63 +173,23 @@ def write_hy_as_pyc(fname):
         fc.write(MAGIC)
 
 
-class MetaLoader(object):
+def _compile_hy(source_text, filename, fullname, *extra):
-    def __init__(self, path):
+    try:
-        self.path = path
+        flags = (__future__.CO_FUTURE_DIVISION |
+                 __future__.CO_FUTURE_PRINT_FUNCTION)
+        return compile(
+            hy_compile(
+                import_buffer_to_hst(source_text.decode('utf-8')), fullname),
+            filename, "exec", flags)
+    except (HyTypeError, LexException) as e:
+        if e.source is None:
+            with open(filename, 'rt') as fp:
+                e.source = fp.read()
+            e.filename = filename
+        raise
+    except Exception:
+        raise
 
-    def is_package(self, fullname):
+polyloader.install(_compile_hy, ['hy'])
-        dirpath = "/".join(fullname.split("."))
+if '' not in sys.path:
-        for pth in sys.path:
+    sys.path.insert(0, '')
-            pth = os.path.abspath(pth)
-            composed_path = "%s/%s/__init__.hy" % (pth, dirpath)
-            if os.path.exists(composed_path):
-                return True
-        return False
-
-    def load_module(self, fullname):
-        if fullname in sys.modules:
-            return sys.modules[fullname]
-
-        if not self.path:
-            return
-
-        sys.modules[fullname] = None
-        mod = import_file_to_module(fullname,
-                                    self.path)
-
-        ispkg = self.is_package(fullname)
-
-        mod.__file__ = self.path
-        mod.__loader__ = self
-        mod.__name__ = fullname
-
-        if ispkg:
-            mod.__path__ = []
-            mod.__package__ = fullname
-        else:
-            mod.__package__ = fullname.rpartition('.')[0]
-
-        sys.modules[fullname] = mod
-        return mod
-
-
-class MetaImporter(object):
-    def find_on_path(self, fullname):
-        fls = ["%s/__init__.hy", "%s.hy"]
-        dirpath = "/".join(fullname.split("."))
-
-        for pth in sys.path:
-            pth = os.path.abspath(pth)
-            for fp in fls:
-                composed_path = fp % ("%s/%s" % (pth, dirpath))
-                if os.path.exists(composed_path):
-                    return composed_path
-
-    def find_module(self, fullname, path=None):
-        path = self.find_on_path(fullname)
-        if path:
-            return MetaLoader(path)
-
-
-sys.meta_path.insert(0, MetaImporter())
-sys.path.insert(0, "")

hy/importer/polyloader/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+import sys
+
+__author__ = 'Kenneth M. "Elf" Sternberg'
+__email__ = 'elf.sternberg@gmail.com'
+__version__ = '0.1.0'
+
+if sys.version_info[0:2] >= (2, 6):
+    from ._python2 import install, reset  # NOQA
+
+if sys.version_info[0] >= 3:
+    from ._python3 import install, reset  # NOQA
+
+__all__ = ['install', 'reset']

hy/importer/polyloader/_python2.py

@@ -0,0 +1,188 @@
+import io
+import os
+import os.path
+import sys
+import imp
+import types
+import pkgutil
+from collections import namedtuple
+
+
+SEP = os.sep
+EXS = os.extsep
+FLS = [('%s' + SEP + '__init__' + EXS + '%s', True),
+       ('%s' + EXS + '%s', False)]
+
+Loader = namedtuple('Loader', 'suffix compiler')
+
+
+class PolyLoader():
+    def __init__(self, fullname, path, is_pkg):
+        self.fullname = fullname
+        self.path = path
+        self.is_package = is_pkg
+
+    def load_module(self, fullname):
+        if fullname in sys.modules:
+            return sys.modules[fullname]
+
+        if fullname != self.fullname:
+            raise ImportError("Load confusion: %s vs %s." %
+                              (fullname, self.fullname))
+
+        matches = [loader for loader in PolyFinder._loader_handlers
+                   if self.path.endswith(loader.suffix)]
+
+        if len(matches) == 0:
+            raise ImportError("%s is not a recognized module?" % fullname)
+
+        if len(matches) > 1:
+            raise ImportError("Multiple possible resolutions for %s: %s" % (
+                fullname, ', '.join([loader.suffix for loader in matches])))
+
+        compiler = matches[0].compiler
+        with io.FileIO(self.path, 'r') as file:
+            source_text = file.read()
+
+        code = compiler(source_text, self.path, fullname)
+
+        module = types.ModuleType(fullname)
+        module.__file__ = self.path
+        module.__name__ = fullname
+        module.__package__ = '.'.join(fullname.split('.')[:-1])
+
+        if self.is_package:
+            module.__path__ = [os.path.dirname(module.__file__)]
+            module.__package__ = fullname
+
+        exec(code, module.__dict__)
+        sys.modules[fullname] = module
+        return module
+
+
+# PolyFinder is an implementation of the Finder class from Python 2.7,
+# with embellishments gleefully copied from Python 3.4.  It supports
+# all the same functionality for non-.py sourcefiles with the added
+# benefit of falling back to Python's default behavior.
+
+# Polyfinder is instantiated by _polyloader_pathhook()
+
+class PolyFinder(object):
+    _loader_handlers = []
+    _installed = False
+
+    def __init__(self, path=None):
+        self.path = path or '.'
+
+    def _pl_find_on_path(self, fullname, path=None):
+        subname = fullname.split(".")[-1]
+        if self.path is None and subname != fullname:
+            return None
+
+        path = os.path.realpath(self.path)
+        for (fp, ispkg) in FLS:
+            for loader in self._loader_handlers:
+                composed_path = fp % (('%s' + SEP + '%s') %
+                                      (path, subname), loader.suffix)
+                if os.path.isdir(composed_path):
+                    r = "Invalid: Directory name ends in recognized suffix"
+                    raise IOError(r)
+                if os.path.isfile(composed_path):
+                    return PolyLoader(fullname, composed_path, ispkg)
+
+        # Fall back onto Python's own methods.
+        try:
+            file, filename, etc = imp.find_module(subname, [path])
+        except ImportError as e:  # NOQA
+            return None
+        return pkgutil.ImpLoader(fullname, file, filename, etc)
+
+    def find_module(self, fullname, path=None):
+        return self._pl_find_on_path(fullname)
+
+    @classmethod
+    def _install(cls, compiler, suffixes):
+        if isinstance(suffixes, basestring):
+            suffixes = [suffixes]
+        suffixes = set(suffixes)
+        overlap = suffixes.intersection(
+            set([suf[0] for suf in imp.get_suffixes()]))
+        if overlap:
+            r = "Override of native Python extensions is not permitted."
+            raise RuntimeError(r)
+        overlap = suffixes.intersection(
+            set([loader.suffix for loader in cls._loader_handlers]))
+        if overlap:
+            # Fail silently
+            return
+        cls._loader_handlers += [Loader(suf, compiler) for suf in suffixes]
+
+    @classmethod
+    def getmodulename(cls, path):
+        filename = os.path.basename(path)
+        suffixes = ([(-len(suf[0]), suf[0])
+                     for suf in imp.get_suffixes()] +
+                    [(-(len(suf[0]) + 1), EXS + suf[0])
+                     for suf in cls._loader_handlers])
+        suffixes.sort()
+        for neglen, suffix in suffixes:
+            if filename[neglen:] == suffix:
+                return filename[:neglen]
+        return None
+
+    def iter_modules(self, prefix=''):
+        if self.path is None or not os.path.isdir(self.path):
+            return
+
+        yielded = {}
+
+        try:
+            filenames = os.listdir(self.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()
+        for fn in filenames:
+            modname = self.getmodulename(fn)
+            if modname == '__init__' or modname in yielded:
+                continue
+
+            path = os.path.join(self.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                try:
+                    dircontents = os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    dircontents = []
+                for fn in dircontents:
+                    subname = self.getmodulename(fn)
+                    if subname == '__init__':
+                        ispkg = True
+                        break
+                else:
+                    continue    # not a package
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield prefix + modname, ispkg
+
+
+def _polyloader_pathhook(path):
+    if not os.path.isdir(path):
+        raise ImportError('Only directories are supported: %s' % path)
+    return PolyFinder(path)
+
+
+def install(compiler, suffixes):
+    if not PolyFinder._installed:
+        sys.path_hooks.append(_polyloader_pathhook)
+        PolyFinder._installed = True
+    PolyFinder._install(compiler, suffixes)
+
+
+def reset():
+    PolyFinder._loader_handlers = []
+    PolyFinder._installed = False

hy/importer/polyloader/_python3.py

@@ -0,0 +1,296 @@
+import os
+import sys
+import marshal
+import pkgutil
+import types
+import _imp
+
+
+# Python 3 refactored importlib several times, resulting in
+# critical pieces of infrastructure moving around or being
+# renamed.  All the NOQA here is to help flake8 get past the
+# "redefinition" complaints.
+
+if sys.version_info[0:2] in [(3, 3), (3, 4)]:
+    from importlib._bootstrap import (  # NOQA
+        cache_from_source, SourceFileLoader,  # NOQA
+        FileFinder, _verbose_message,  # NOQA
+        _get_supported_file_loaders, _relax_case,  # NOQA
+        _w_long, _code_type)  # NOQA
+
+
+if sys.version_info[0:2] in [(3, 3)]:
+    from importlib._bootstrap import _MAGIC_BYTES as MAGIC_NUMBER  # NOQA
+
+if sys.version_info[0:2] == (3, 4):
+    from importlib._bootstrap import _validate_bytecode_header, MAGIC_NUMBER  # NOQA
+
+if sys.version_info[0:2] >= (3, 5):
+    from importlib.machinery import SourceFileLoader, FileFinder  # NOQA
+    from importlib._bootstrap import _verbose_message  # NOQA
+    from importlib._bootstrap_external import (  # NOQA
+        _w_long, _code_type, cache_from_source,  # NOQA
+        _validate_bytecode_header,  # NOQA
+        MAGIC_NUMBER, _relax_case,  # NOQA
+        _get_supported_file_loaders)  # NOQA
+
+SEP = os.sep
+EXS = os.extsep
+FLS = [('%s' + SEP + '__init__' + EXS + '%s', True),
+       ('%s' + EXS + '%s', False)]
+
+
+def _suffixer(loaders):
+    return [(suffix, loader)
+            for (loader, suffixes) in loaders
+            for suffix in suffixes]
+
+
+class _PolySourceFileLoader(SourceFileLoader):
+    _compiler = None
+
+    def _poly_bytes_from_bytecode(self, fullname, data, path, st):
+        if hasattr(self, '_bytes_from_bytecode'):
+            return self._bytes_from_bytecode(fullname, data,
+                                             path, st)
+        self_module = sys.modules[__name__]
+        if hasattr(self_module, '_validate_bytecode_header'):
+            return _validate_bytecode_header(data, source_stats=st,
+                                             name=fullname, path=path)
+        raise ImportError("No bytecode handler found loading.")
+
+    # All this just to change one line.
+    def get_code(self, fullname):
+        source_path = self.get_filename(fullname)
+        source_mtime = None
+        try:
+            bytecode_path = cache_from_source(source_path)
+        except NotImplementedError:
+            bytecode_path = None
+        else:
+            try:
+                st = self.path_stats(source_path)
+            except NotImplementedError:
+                pass
+            else:
+                source_mtime = int(st['mtime'])
+                try:
+                    data = self.get_data(bytecode_path)
+                except IOError:
+                    pass
+                else:
+                    try:
+                        bytes_data = self._poly_bytes_from_bytecode(
+                            fullname, data,
+                            bytecode_path,
+                            st)
+                    except (ImportError, EOFError):
+                        pass
+                    else:
+                        _verbose_message(
+                            '{} matches {}',
+                            bytecode_path,
+                            source_path)
+                        found = marshal.loads(bytes_data)
+                        if isinstance(found, _code_type):
+                            _imp._fix_co_filename(found, source_path)
+                            _verbose_message(
+                                'code object from {}',
+                                bytecode_path)
+                            return found
+                        else:
+                            msg = "Non-code object in {}"
+                            raise ImportError(
+                                msg.format(bytecode_path),
+                                name=fullname,
+                                path=bytecode_path)
+        source_bytes = self.get_data(source_path)
+        code_object = self._compiler(source_bytes, source_path, fullname)
+        _verbose_message('code object from {}', source_path)
+        if (not sys.dont_write_bytecode and
+                bytecode_path is not None and
+                source_mtime is not None):
+            data = bytearray(MAGIC_NUMBER)
+            data.extend(_w_long(source_mtime))
+            data.extend(_w_long(len(source_bytes)))
+            data.extend(marshal.dumps(code_object))
+            try:
+                self._cache_bytecode(source_path, bytecode_path, data)
+                _verbose_message('wrote {!r}', bytecode_path)
+            except NotImplementedError:
+                pass
+        return code_object
+
+
+class PolyFileFinder(FileFinder):
+    '''The poly version of FileFinder supports the addition of loaders
+       after initialization.  That's pretty much the whole point of the
+       PolyLoader mechanism.'''
+
+    _native_loaders = []
+    _custom_loaders = []
+
+    def __init__(self, path):
+        # Base (directory) path
+        self.path = path or '.'
+        self._path_mtime = -1
+        self._path_cache = set()
+        self._relaxed_path_cache = set()
+
+    @property
+    def _loaders(self):
+        return self._custom_loaders + list(self._native_loaders)
+
+    @classmethod
+    def _install(cls, compiler, suffixes):
+        if not suffixes:
+            return
+        if isinstance(suffixes, str):
+            suffixes = [suffixes]
+        suffixset = set(suffixes)
+        overlap = suffixset.intersection(
+            set([suf[0] for suf in cls._native_loaders]))
+        if overlap:
+            r = "Override of native Python extensions is not permitted."
+            raise RuntimeError(r)
+        overlap = suffixset.intersection(
+            set([loader[0] for loader in cls._custom_loaders]))
+        if overlap:
+            # Fail silently
+            return
+
+        newloaderclassname = (
+            suffixes[0].lower().capitalize() +
+            str(_PolySourceFileLoader).rpartition('.')[2][1:])
+        if isinstance(compiler, types.FunctionType):
+            newloader = type(newloaderclassname, (_PolySourceFileLoader,),
+                             dict(_compiler=staticmethod(compiler)))
+        else:
+            newloader = type(newloaderclassname, (_PolySourceFileLoader,),
+                             dict(_compiler=compiler))
+        cls._custom_loaders += [(EXS + suffix, newloader)
+                                for suffix in suffixset]
+
+    @classmethod
+    def getmodulename(cls, path):
+        filename = os.path.basename(path)
+        suffixes = ([(-len(suf[0]), suf[0]) for suf in cls._native_loaders] +
+                    [(-len(suf[0]), suf[0]) for suf in cls._custom_loaders])
+        suffixes.sort()
+        for neglen, suffix in suffixes:
+            if filename[neglen:] == suffix:
+                return filename[:neglen]
+        return None
+
+    def find_loader(self, fullname):
+        """Try to find a loader for the specified module, or the namespace
+        package portions. Returns (loader, list-of-portions)."""
+        is_namespace = False
+        tail_module = fullname.rpartition('.')[2]
+        try:
+            mtime = os.stat(self.path).st_mtime
+        except OSError:
+            mtime = -1
+        if mtime != self._path_mtime:
+            self._fill_cache()
+            self._path_mtime = mtime
+        # tail_module keeps the original casing, for __file__ and friends
+        if _relax_case():
+            cache = self._relaxed_path_cache
+            cache_module = tail_module.lower()
+        else:
+            cache = self._path_cache
+            cache_module = tail_module
+        # Check if the module is the name of a directory (and thus a package).
+        if cache_module in cache:
+            base_path = os.path.join(self.path, tail_module)
+            if os.path.isdir(base_path):
+                for suffix, loader in self._loaders:
+                    init_filename = '__init__' + suffix
+                    full_path = os.path.join(base_path, init_filename)
+                    if os.path.isfile(full_path):
+                        return (loader(fullname, full_path), [base_path])
+                else:
+                    # A namespace package, return the path if we don't also
+                    #  find a module in the next section.
+                    is_namespace = True
+        # Check for a file w/ a proper suffix exists.
+        for suffix, loader in self._loaders:
+            full_path = os.path.join(self.path, tail_module + suffix)
+            _verbose_message('trying {}'.format(full_path), verbosity=2)
+            if cache_module + suffix in cache:
+                if os.path.isfile(full_path):
+                    return (loader(fullname, full_path), [])
+        if is_namespace:
+            _verbose_message('possible namespace for {}'.format(base_path))
+            return (None, [base_path])
+        return (None, [])
+
+    @classmethod
+    def path_hook(cls, *loader_details):
+        cls._native_loaders = loader_details
+
+        def path_hook_for_PolyFileFinder(path):
+            if not os.path.isdir(path):
+                raise ImportError("only directories are supported", path=path)
+            return PolyFileFinder(path)
+        return path_hook_for_PolyFileFinder
+
+
+def _poly_file_finder_modules(importer, prefix=''):
+    if importer.path is None or not os.path.isdir(importer.path):
+        return
+
+    yielded = {}
+
+    try:
+        filenames = os.listdir(importer.path)
+    except OSError:
+        # ignore unreadable directories like import does
+        filenames = []
+    filenames.sort()
+    for fn in filenames:
+        modname = importer.getmodulename(fn)
+        if modname == '__init__' or modname in yielded:
+            continue
+
+        path = os.path.join(importer.path, fn)
+        ispkg = False
+
+        if not modname and os.path.isdir(path) and '.' not in fn:
+            modname = fn
+            try:
+                dircontents = os.listdir(path)
+            except OSError:
+                # ignore unreadable directories like import does
+                dircontents = []
+            for fn in dircontents:
+                subname = importer.getmodulename(fn)
+                if subname == '__init__':
+                    ispkg = True
+                    break
+            else:
+                continue    # not a package
+
+        if modname and '.' not in modname:
+            yielded[modname] = 1
+            yield prefix + modname, ispkg
+
+
+def install(compiler, suffixes):
+    filefinder = [(f, i) for i, f in enumerate(sys.path_hooks)
+                  if repr(f).find('.path_hook_for_FileFinder') != -1]
+    if filefinder:
+        filefinder, fpos = filefinder[0]
+        sys.path_hooks[fpos] = PolyFileFinder.path_hook(
+            *(_suffixer(_get_supported_file_loaders())))
+        sys.path_importer_cache = {}
+        pkgutil.iter_importer_modules.register(
+            PolyFileFinder,
+            _poly_file_finder_modules)
+
+    PolyFileFinder._install(compiler, suffixes)
+
+
+def reset():
+    PolyFileFinder._custom_loaders = []

tests/importer/test_importer.py

@@ -1,6 +1,5 @@
-from hy.importer import import_file_to_module, import_buffer_to_ast, MetaLoader
+from hy.importer import import_file_to_module, import_buffer_to_ast
 from hy.errors import HyTypeError
-import os
 import ast
 
 
@@ -17,12 +16,9 @@ def test_stringer():
 
 
 def test_imports():
-    path = os.getcwd() + "/tests/resources/importer/a.hy"
-    testLoader = MetaLoader(path)
-
     def _import_test():
         try:
-            return testLoader.load_module("tests.resources.importer.a")
+            import tests.resources.importer.a  # NOQA
         except:
             return "Error"