From b71962bbaaeaf2ae7b80c3dff69bcfcb9d3d1fc9 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Thu, 6 Apr 2017 16:15:13 -0700 Subject: [PATCH 1/5] Remove unused internal fn import_file_to_globals --- hy/importer.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index c3c0192..0a5f223 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -89,15 +89,6 @@ def import_file_to_module(module_name, fpath): return mod -def import_file_to_globals(env, module_name, fpath): - """ Import content from fpath and puts it into the dict provided - (e.g., for use in a REPL) - """ - mod = import_file_to_module(module_name, fpath) - for k, v in mod.__dict__.items(): - env[k] = v - - def import_buffer_to_module(module_name, buf): try: _ast = import_buffer_to_ast(buf, module_name) From 7a53fdb18001adf5b10c56d456d5f8a2834b641a Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sat, 8 Apr 2017 09:52:18 -0700 Subject: [PATCH 2/5] Fold .coveragerc into setup.cfg --- .coveragerc | 13 ------------- setup.cfg | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 1e3173b..0000000 --- a/.coveragerc +++ /dev/null @@ -1,13 +0,0 @@ -[run] -omit = - */python?.?/* - */lib-python/?.?/*.py - */lib_pypy/_*.py - */site-packages/nose/* - */pypy/* - - -[report] -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index b8bb4f9..d1d1807 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,3 +6,16 @@ nocapture=1 [wheel] universal = 1 + +[coverage:run] +omit = + */python?.?/* + */lib-python/?.?/*.py + */lib_pypy/_*.py + */site-packages/nose/* + */pypy/* + +[coverage:report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover From 2b11b9be20c20d53b6f410ad5b9e3235e290bf51 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Sun, 9 Apr 2017 17:27:51 -0700 Subject: [PATCH 3/5] Automatically read and write bytecode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing or executing a Hy file now loads the byte-compiled version if it exists and is up to date, and if not, the source is byte-compiled after it's parsed. This change can speed up Hy a lot. Here are some examples comparing run times of the current master (491b474e) to this commit, on my laptop with Python 3.6: - `nosetests --exclude='test_bin'` goes from 3.8 s to 0.7 s (a 5-fold speedup) - `hy -c '(print "hello world")` goes from 0.47 s to 0.20 s (a 2-fold speedup) - Rogue TV's startup goes from 3.6 s to 0.4 s (a 9-fold speedup) Accompanying changes include: - `setup.py` now creates and installs bytecode for `hy.core`, `hy.contrib`, and `hy.extra`. - The `hyc` command under Python 3 now creates bytecode in `__pycache__`, as usual for Python 3, instead of putting the `.pyc` right next to the source file like Python 2 does. I've removed a test of `hy.extra.anaphoric.a-if` that triggers #1268 when the test file is byte-compiled and then hits some weird `macroexpand` bug or something when I try to work around that—Nose crashes when trying to produce an error message, and I can't seem to replicate the bug without Nose. --- NEWS | 4 + hy/importer.py | 169 ++++++++++++++++---------- setup.cfg | 3 + setup.py | 19 ++- tests/importer/test_importer.py | 23 +++- tests/importer/test_pyc.py | 12 +- tests/native_tests/extra/anaphoric.hy | 5 +- tests/resources/bin/bytecompile.hy | 5 + tests/test_bin.py | 52 +++++++- 9 files changed, 213 insertions(+), 79 deletions(-) create mode 100644 tests/resources/bin/bytecompile.hy diff --git a/NEWS b/NEWS index 01d139b..88991c1 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,10 @@ Changes from 0.12.1 * `let` has been removed. Python's scoping rules do not make a proper implementation of it possible. Use `setv` instead. * `lambda` has been removed, but `fn` now does exactly what `lambda` did + * Importing or executing a Hy file automatically byte-compiles it, or loads + a byte-compiled version if it exists and is up to date. This brings big + speed boosts, even for one-liners, because Hy no longer needs to recompile + its standard library for every startup. * Added bytestring literals, which create `bytes` objects under Python 3 and `str` objects under Python 2 * Commas and underscores are allowed in numeric literals diff --git a/hy/importer.py b/hy/importer.py index 0a5f223..cea5b46 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -25,14 +25,16 @@ from hy.lex import tokenize, LexException from hy.errors import HyIOError from io import open +import re import marshal +import struct import imp import sys import ast import os import __future__ -from hy._compat import PY3, PY33, MAGIC, builtins, long_type, wr_long +from hy._compat import PY3, PY33, PY34, MAGIC, builtins, long_type, wr_long from hy._compat import string_types @@ -68,25 +70,77 @@ def import_file_to_ast(fpath, module_name): return hy_compile(import_file_to_hst(fpath), module_name) -def import_file_to_module(module_name, fpath): - """Import content from fpath and puts it into a Python module. +def import_file_to_module(module_name, fpath, loader=None): + """Import Hy source from fpath and put it into a Python module. - Returns the module.""" + If there's an up-to-date byte-compiled version of this module, load that + instead. Otherwise, byte-compile the module once we're done loading it, if + we can. + + Return the module.""" + + module = None + + bytecode_path = get_bytecode_path(fpath) try: - _ast = import_file_to_ast(fpath, module_name) - mod = imp.new_module(module_name) - mod.__file__ = fpath - eval(ast_compile(_ast, fpath, "exec"), mod.__dict__) - except (HyTypeError, LexException) as e: - if e.source is None: - with open(fpath, 'rt') as fp: - e.source = fp.read() - e.filename = fpath - raise - except Exception: - sys.modules.pop(module_name, None) - raise - return mod + source_mtime = int(os.stat(fpath).st_mtime) + with open(bytecode_path, 'rb') as bc_f: + # To get the bytecode file's internal timestamp, take the 4 bytes + # after the first 4 bytes and interpret them as a little-endian + # 32-bit integer. + bytecode_mtime = struct.unpack('= source_mtime: + # It's a cache hit. Load the byte-compiled version. + if PY3: + # As of Python 3.6, imp.load_compiled still exists, but it's + # deprecated. So let's use SourcelessFileLoader instead. + from importlib.machinery import SourcelessFileLoader + module = (SourcelessFileLoader(module_name, bytecode_path). + load_module(module_name)) + else: + module = imp.load_compiled(module_name, bytecode_path) + + if not module: + # It's a cache miss, so load from source. + sys.modules[module_name] = None + try: + _ast = import_file_to_ast(fpath, module_name) + module = imp.new_module(module_name) + module.__file__ = fpath + code = ast_compile(_ast, fpath, "exec") + try: + write_code_as_pyc(fpath, code) + except (IOError, OSError): + # We failed to save the bytecode, probably because of a + # permissions issue. The user only asked to import the file, so + # don't bug them about it. + pass + eval(code, module.__dict__) + except (HyTypeError, LexException) as e: + if e.source is None: + with open(fpath, 'rt') as fp: + e.source = fp.read() + e.filename = fpath + raise + except Exception: + sys.modules.pop(module_name, None) + raise + sys.modules[module_name] = module + module.__file__ = fpath + module.__name__ = module_name + + if loader: + module.__loader__ = loader + if is_package(module_name): + module.__path__ = [] + module.__package__ = module_name + else: + module.__package__ = module_name.rpartition('.')[0] + + return module def import_buffer_to_module(module_name, buf): @@ -138,47 +192,34 @@ def hy_eval(hytree, namespace, module_name, ast_callback=None): def write_hy_as_pyc(fname): - with open(fname, 'U') as f: - try: - st = os.fstat(f.fileno()) - except AttributeError: - st = os.stat(fname) - timestamp = long_type(st.st_mtime) - _ast = import_file_to_ast(fname, os.path.basename(os.path.splitext(fname)[0])) code = ast_compile(_ast, fname, "exec") - cfile = "%s.pyc" % fname[:-len(".hy")] + write_code_as_pyc(fname, code) - open_ = builtins.open - with open_(cfile, 'wb') as fc: - if PY3: - fc.write(b'\0\0\0\0') - else: - fc.write('\0\0\0\0') +def write_code_as_pyc(fname, code): + st = os.stat(fname) + timestamp = long_type(st.st_mtime) + + cfile = get_bytecode_path(fname) + try: + os.makedirs(os.path.dirname(cfile)) + except (IOError, OSError): + pass + + with builtins.open(cfile, 'wb') as fc: + fc.write(MAGIC) wr_long(fc, timestamp) if PY33: wr_long(fc, st.st_size) marshal.dump(code, fc) - fc.flush() - fc.seek(0, 0) - fc.write(MAGIC) class MetaLoader(object): def __init__(self, path): self.path = path - def is_package(self, fullname): - dirpath = "/".join(fullname.split(".")) - for pth in sys.path: - pth = os.path.abspath(pth) - composed_path = "%s/%s/__init__.hy" % (pth, dirpath) - if os.path.exists(composed_path): - return True - return False - def load_module(self, fullname): if fullname in sys.modules: return sys.modules[fullname] @@ -186,24 +227,7 @@ class MetaLoader(object): if not self.path: return - sys.modules[fullname] = None - mod = import_file_to_module(fullname, - self.path) - - ispkg = self.is_package(fullname) - - mod.__file__ = self.path - mod.__loader__ = self - mod.__name__ = fullname - - if ispkg: - mod.__path__ = [] - mod.__package__ = fullname - else: - mod.__package__ = fullname.rpartition('.')[0] - - sys.modules[fullname] = mod - return mod + return import_file_to_module(fullname, self.path, self) class MetaImporter(object): @@ -226,3 +250,24 @@ class MetaImporter(object): sys.meta_path.insert(0, MetaImporter()) sys.path.insert(0, "") + + +def is_package(module_name): + mpath = os.path.join(*module_name.split(".")) + for path in map(os.path.abspath, sys.path): + if os.path.exists(os.path.join(path, mpath, "__init__.hy")): + return True + return False + + +def get_bytecode_path(source_path): + if PY34: + import importlib.util + return importlib.util.cache_from_source(source_path) + elif hasattr(imp, "cache_from_source"): + return imp.cache_from_source(source_path) + else: + # If source_path has a file extension, replace it with ".pyc". + # Otherwise, just append ".pyc". + d, f = os.path.split(source_path) + return os.path.join(d, re.sub(r"(?:\.[^.]+)?\Z", ".pyc", f)) diff --git a/setup.cfg b/setup.cfg index d1d1807..31c8825 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,6 @@ omit = exclude_lines = # Have to re-enable the standard pragma pragma: no cover +# We want ignore_errors so we don't get NoSource warnings for loading +# byte-compiled Hy modules. +ignore_errors = True diff --git a/setup.py b/setup.py index 75a61c3..546cfa1 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ import runpy import subprocess from setuptools import find_packages, setup +from setuptools.command.install import install os.chdir(os.path.split(os.path.abspath(__file__))[0]) @@ -48,6 +49,17 @@ long_description = """Hy is a Python <--> Lisp layer. It helps make things work nicer, and lets Python and the Hy lisp variant play nice together. """ +class Install(install): + def run(self): + # Import each Hy module to ensure it's compiled. + import os, importlib + for dirpath, _, filenames in os.walk("hy"): + for filename in filenames: + if filename.endswith(".hy"): + importlib.import_module( + dirpath.replace("/", ".") + "." + filename[:-len(".hy")]) + install.run(self) + install_requires = ['rply>=0.7.0', 'astor>=0.5', 'clint>=0.4'] if sys.version_info[:2] < (2, 7): install_requires.append('argparse>=1.2.1') @@ -61,6 +73,7 @@ setup( name=PKG, version=__version__, install_requires=install_requires, + cmdclass=dict(install=Install), entry_points={ 'console_scripts': [ 'hy = hy.cmdline:hy_main', @@ -73,9 +86,9 @@ setup( }, packages=find_packages(exclude=['tests*']), package_data={ - 'hy.contrib': ['*.hy'], - 'hy.core': ['*.hy'], - 'hy.extra': ['*.hy'], + 'hy.contrib': ['*.hy', '__pycache__/*'], + 'hy.core': ['*.hy', '__pycache__/*'], + 'hy.extra': ['*.hy', '__pycache__/*'], }, author="Paul Tagliamonte", author_email="tag@pault.ag", diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index edfbb5a..d33cd1e 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -1,7 +1,9 @@ -from hy.importer import import_file_to_module, import_buffer_to_ast, MetaLoader +from hy.importer import (import_file_to_module, import_buffer_to_ast, + MetaLoader, get_bytecode_path) from hy.errors import HyTypeError import os import ast +import tempfile def test_basics(): @@ -11,7 +13,6 @@ def test_basics(): def test_stringer(): - "Make sure the basics of the importer work" _ast = import_buffer_to_ast("(defn square [x] (* x x))", '') assert type(_ast.body[0]) == ast.FunctionDef @@ -41,3 +42,21 @@ def test_import_error_reporting(): assert _import_error_test() == "Error reported" assert _import_error_test() is not None + + +def test_import_autocompiles(): + "Test that (import) byte-compiles the module." + + f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False) + f.write(b'(defn pyctest [s] (+ "X" s "Y"))') + f.close() + + try: + os.remove(get_bytecode_path(f.name)) + except (IOError, OSError): + pass + import_file_to_module("mymodule", f.name) + assert os.path.exists(get_bytecode_path(f.name)) + + os.remove(f.name) + os.remove(get_bytecode_path(f.name)) diff --git a/tests/importer/test_pyc.py b/tests/importer/test_pyc.py index 47d5e51..1b61197 100644 --- a/tests/importer/test_pyc.py +++ b/tests/importer/test_pyc.py @@ -1,20 +1,20 @@ import os import imp import tempfile -from hy.importer import write_hy_as_pyc +from hy.importer import write_hy_as_pyc, get_bytecode_path def test_pyc(): """Test pyc compilation.""" f = tempfile.NamedTemporaryFile(suffix='.hy', delete=False) - f.write(b'(defn pyctest [s] s)') + f.write(b'(defn pyctest [s] (+ "X" s "Y"))') f.close() write_hy_as_pyc(f.name) - os.unlink(f.name) + os.remove(f.name) - cfile = "%s.pyc" % f.name[:-len(".hy")] + cfile = get_bytecode_path(f.name) mod = imp.load_compiled('pyc', cfile) - os.unlink(cfile) + os.remove(cfile) - assert mod.pyctest('Foo') == 'Foo' + assert mod.pyctest('Foo') == 'XFooY' diff --git a/tests/native_tests/extra/anaphoric.hy b/tests/native_tests/extra/anaphoric.hy index 2c0b73a..0d2cbaf 100644 --- a/tests/native_tests/extra/anaphoric.hy +++ b/tests/native_tests/extra/anaphoric.hy @@ -36,10 +36,7 @@ (defn test-ap-if [] "NATIVE: testing anaphoric if" (ap-if True (assert-true it)) - (ap-if False True (assert-false it)) - (try (macroexpand '(ap-if True)) - (except [HyMacroExpansionError] True) - (else (assert False)))) + (ap-if False True (assert-false it))) (defn test-ap-each [] "NATIVE: testing anaphoric each" diff --git a/tests/resources/bin/bytecompile.hy b/tests/resources/bin/bytecompile.hy new file mode 100644 index 0000000..bf0b0d0 --- /dev/null +++ b/tests/resources/bin/bytecompile.hy @@ -0,0 +1,5 @@ +(defmacro m [] + (print "Hello from macro") + "boink") + +(print "The macro returned:" (m)) diff --git a/tests/test_bin.py b/tests/test_bin.py index 0c748de..b0487eb 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -25,6 +25,7 @@ import os import subprocess import re from hy._compat import PY3 +from hy.importer import get_bytecode_path hy_dir = os.environ.get('HY_DIR', '') @@ -55,6 +56,16 @@ def run_cmd(cmd, stdin_data=None, expect=0): return stdout, stderr +def rm(fpath): + try: + os.remove(fpath) + except (IOError, OSError): + try: + os.rmdir(fpath) + except (IOError, OSError): + pass + + def test_bin_hy(): run_cmd("hy", "") @@ -190,9 +201,11 @@ def test_bin_hyc(): output, _ = run_cmd("hyc -h") assert "usage" in output - output, _ = run_cmd("hyc tests/resources/argparse_ex.hy") + path = "tests/resources/argparse_ex.hy" + output, _ = run_cmd("hyc " + path) assert "Compiling" in output - assert os.path.exists("tests/resources/argparse_ex.pyc") + assert os.path.exists(get_bytecode_path(path)) + rm(get_bytecode_path(path)) def test_bin_hyc_missing_file(): @@ -243,6 +256,41 @@ def test_bin_hy_no_main(): assert "This Should Still Work" in output +def test_bin_hy_byte_compile(): + + modname = "tests.resources.bin.bytecompile" + fpath = modname.replace(".", "/") + ".hy" + + for can_byte_compile in [True, False]: + for cmd in ["hy " + fpath, + "hy -m " + modname, + "hy -c '(import {})'".format(modname)]: + + rm(get_bytecode_path(fpath)) + + if not can_byte_compile: + # Keep Hy from being able to byte-compile the module by + # creating a directory at the target location. + os.mkdir(get_bytecode_path(fpath)) + + # Whether or not we can byte-compile the module, we should be able + # to run it. + output, _ = run_cmd(cmd) + assert "Hello from macro" in output + assert "The macro returned: boink" in output + + if can_byte_compile: + # That should've byte-compiled the module. + assert os.path.exists(get_bytecode_path(fpath)) + + # When we run the same command again, and we've byte-compiled the + # module, the byte-compiled version should be run instead of the + # source, in which case the macro shouldn't be run. + output, _ = run_cmd(cmd) + assert ("Hello from macro" in output) ^ can_byte_compile + assert "The macro returned: boink" in output + + def test_bin_hy_module_main(): output, _ = run_cmd("hy -m tests.resources.bin.main") assert "Hello World" in output From 36324e94998a20a16b5c38801a94fdaa0aa3f368 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Mon, 10 Apr 2017 11:30:18 -0700 Subject: [PATCH 4/5] Work around a Nose import bug on PyPy --- hy/importer.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index cea5b46..4a6033d 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -30,6 +30,7 @@ import marshal import struct import imp import sys +import platform import ast import os import __future__ @@ -111,13 +112,17 @@ def import_file_to_module(module_name, fpath, loader=None): module = imp.new_module(module_name) module.__file__ = fpath code = ast_compile(_ast, fpath, "exec") - try: - write_code_as_pyc(fpath, code) - except (IOError, OSError): - # We failed to save the bytecode, probably because of a - # permissions issue. The user only asked to import the file, so - # don't bug them about it. - pass + if not (platform.python_implementation() == 'PyPy' and + 'nosetests' in sys.argv[0] and + is_package(module_name)): + # Nose can generate spurious errors in this specific situation. + try: + write_code_as_pyc(fpath, code) + except (IOError, OSError): + # We failed to save the bytecode, probably because of a + # permissions issue. The user only asked to import the + # file, so don't bug them about it. + pass eval(code, module.__dict__) except (HyTypeError, LexException) as e: if e.source is None: From a87b23b4e80c5daaff456c12332b1e77aff18601 Mon Sep 17 00:00:00 2001 From: Kodi Arfer Date: Tue, 11 Apr 2017 18:53:35 -0700 Subject: [PATCH 5/5] Check the magic number of bytecode files --- hy/importer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hy/importer.py b/hy/importer.py index 4a6033d..bf71169 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -86,14 +86,16 @@ def import_file_to_module(module_name, fpath, loader=None): try: source_mtime = int(os.stat(fpath).st_mtime) with open(bytecode_path, 'rb') as bc_f: - # To get the bytecode file's internal timestamp, take the 4 bytes - # after the first 4 bytes and interpret them as a little-endian - # 32-bit integer. - bytecode_mtime = struct.unpack('= source_mtime: + if bytecode_magic == MAGIC and bytecode_mtime >= source_mtime: # It's a cache hit. Load the byte-compiled version. if PY3: # As of Python 3.6, imp.load_compiled still exists, but it's