Merge pull request #1678 from brandonwillard/run-ambiguous-files-as-hy

Allow `runpy` to consider non-standard Hy source extensions

hy/cmdline.py

@@ -19,7 +19,7 @@ import astor.code_gen
 import hy
 from hy.lex import LexException, PrematureEndOfInput, mangle
 from hy.compiler import HyTypeError, hy_compile
-from hy.importer import hy_eval, hy_parse
+from hy.importer import hy_eval, hy_parse, runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
@@ -352,7 +352,7 @@ def cmdline_handler(scriptname, argv):
 
             try:
                 sys.argv = options.args
-                runpy.run_path(filename, run_name='__main__')
+                runhy.run_path(filename, run_name='__main__')
                 return 0
             except FileNotFoundError as e:
                 print("hy: Can't open file '{0}': [Errno {1}] {2}".format(

hy/importer.py

@@ -11,12 +11,13 @@ import inspect
 import pkgutil
 import re
 import io
-import runpy
 import types
 import tempfile
 import importlib
 import __future__
 
+from functools import partial
+
 from hy.errors import HyTypeError
 from hy.compiler import hy_compile
 from hy.lex import tokenize, LexException
@@ -166,46 +167,62 @@ def cache_from_source(source_path):
         return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f))
 
 
-def _get_code_from_file(run_name, fname=None):
+def _hy_code_from_file(filename, loader_type=None):
-    """A patch of `runpy._get_code_from_file` that will also compile Hy
+    """Use PEP-302 loader to produce code for a given Hy source file."""
-    code.
+    full_fname = os.path.abspath(filename)
+    fname_path, fname_file = os.path.split(full_fname)
+    modname = os.path.splitext(fname_file)[0]
+    sys.path.insert(0, fname_path)
+    try:
+        if loader_type is None:
+            loader = pkgutil.get_loader(modname)
+        else:
+            loader = loader_type(modname, full_fname)
+        code = loader.get_code(modname)
+    finally:
+        sys.path.pop(0)
 
-    This version will read and cache bytecode for Hy files.  It operates
+    return code
-    normally otherwise.
+
+
+def _get_code_from_file(run_name, fname=None,
+                        hy_src_check=lambda x: x.endswith('.hy')):
+    """A patch of `runpy._get_code_from_file` that will also run and cache Hy
+    code.
     """
     if fname is None and run_name is not None:
         fname = run_name
 
-    if fname.endswith('.hy'):
+    # Check for bytecode first.  (This is what the `runpy` version does!)
-        full_fname = os.path.abspath(fname)
+    with open(fname, "rb") as f:
-        fname_path, fname_file = os.path.split(full_fname)
+        code = pkgutil.read_code(f)
-        modname = os.path.splitext(fname_file)[0]
+
-        sys.path.insert(0, fname_path)
+    if code is None:
-        try:
+        if hy_src_check(fname):
-            loader = pkgutil.get_loader(modname)
+            code = _hy_code_from_file(fname, loader_type=HyLoader)
-            code = loader.get_code(modname)
+        else:
-        finally:
+            # Try normal source
-            sys.path.pop(0)
-    else:
-        with open(fname, "rb") as f:
-            code = pkgutil.read_code(f)
-        if code is None:
             with open(fname, "rb") as f:
+                # This code differs from `runpy`'s only in that we
+                # force decoding into UTF-8.
                 source = f.read().decode('utf-8')
             code = compile(source, fname, 'exec')
 
     return (code, fname) if PY3 else code
 
 
-_runpy_get_code_from_file = runpy._get_code_from_file
-runpy._get_code_from_file = _get_code_from_file
-
 if PY3:
     importlib.machinery.SOURCE_SUFFIXES.insert(0, '.hy')
     _py_source_to_code = importlib.machinery.SourceFileLoader.source_to_code
 
+    def _could_be_hy_src(filename):
+        return (os.path.isfile(filename) and
+            (filename.endswith('.hy') or
+             not any(filename.endswith(ext)
+                     for ext in importlib.machinery.SOURCE_SUFFIXES[1:])))
+
     def _hy_source_to_code(self, data, path, _optimize=-1):
-        if os.path.isfile(path) and path.endswith('.hy'):
+        if _could_be_hy_src(path):
             source = data.decode("utf-8")
             try:
                 hy_tree = hy_parse(source)
@@ -242,12 +259,17 @@ else:
 
     from pkgutil import ImpImporter, ImpLoader
 
+    def _could_be_hy_src(filename):
+        return (filename.endswith('.hy') or
+                (os.path.isfile(filename) and
+                 not any(filename.endswith(s[0]) for s in imp.get_suffixes())))
+
     class HyLoader(ImpLoader, object):
         def __init__(self, fullname, filename, fileobj=None, etc=None):
             """This constructor is designed for some compatibility with
             SourceFileLoader."""
             if etc is None and filename is not None:
-                if filename.endswith('.hy'):
+                if _could_be_hy_src(filename):
                     etc = ('.hy', 'U', imp.PY_SOURCE)
                     if fileobj is None:
                         fileobj = io.open(filename, 'rU', encoding='utf-8')
@@ -477,7 +499,7 @@ else:
 
             try:
                 flags = None
-                if filename.endswith('.hy'):
+                if _could_be_hy_src(filename):
                     hy_tree = hy_parse(source_str)
                     source = hy_compile(hy_tree, '<hyc_compile>')
                     flags = hy_ast_compile_flags
@@ -530,3 +552,18 @@ else:
         return cfile
 
     py_compile.compile = hyc_compile
+
+
+# We create a separate version of runpy, "runhy", that prefers Hy source over
+# Python.
+runhy = importlib.import_module('runpy')
+
+runhy._get_code_from_file = partial(_get_code_from_file,
+                                    hy_src_check=_could_be_hy_src)
+
+del sys.modules['runpy']
+
+runpy = importlib.import_module('runpy')
+
+_runpy_get_code_from_file = runpy._get_code_from_file
+runpy._get_code_from_file = _get_code_from_file

tests/resources/no_extension

@@ -0,0 +1,2 @@
+#!/usr/bin/env hy
+(print "This Should Still Work")

tests/test_bin.py

@@ -382,3 +382,9 @@ def test_bin_hy_module_no_main():
 def test_bin_hy_sys_executable():
     output, _ = run_cmd("hy -c '(do (import sys) (print sys.executable))'")
     assert output.strip().endswith('/hy')
+
+
+def test_bin_hy_file_no_extension():
+    """Confirm that a file with no extension is processed as Hy source"""
+    output, _ = run_cmd("hy tests/resources/no_extension")
+    assert "This Should Still Work" in output