Add build environment caching (pypa#486)

docs/history.md

@@ -18,6 +18,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ***Added:***
 
 - Add `project` command group to view details about the project like PEP 621 metadata
+- Build environments for the `virtual` environment type are now cached for improved performance
+- Add `build_environment_exists` method to the environment interface for implementations that cache the build environment
 - Support Bash on Windows for the `shell` command
 - The `setuptools` migration script no longer modifies the formatting of existing `pyproject.toml` configuration
 - Bump the minimum supported version of Hatchling to 1.9.0

docs/plugins/environment/reference.md

@@ -48,6 +48,7 @@ Whenever an environment is used, the following logic is performed:
       - dependencies_in_sync
       - sync_dependencies
       - build_environment
+      - build_environment_exists
       - get_build_process
       - construct_build_command
       - command_context

src/hatch/cli/build/__init__.py

@@ -95,7 +95,9 @@ def get_version_api(self):
             with environment.get_env_vars(), EnvVars(env_vars):
                 dependencies.extend(builder.config.dependencies)
 
-            with app.status_waiting('Setting up build environment') as status:
+            with app.status_waiting(
+                'Setting up build environment', condition=not environment.build_environment_exists()
+            ) as status:
                 with environment.build_environment(dependencies) as build_environment:
                     status.stop()
 

src/hatch/cli/env/prune.py

@@ -35,6 +35,6 @@ def prune(app):
         except Exception:
             continue
 
-        if environment.exists():
+        if environment.exists() or environment.build_environment_exists():
             with app.status_waiting(f'Removing environment: {env_name}'):
                 environment.remove()

src/hatch/cli/env/remove.py

@@ -27,6 +27,6 @@ def remove(ctx, env_name):
         except Exception:
             continue
 
-        if environment.exists():
+        if environment.exists() or environment.build_environment_exists():
             with app.status_waiting(f'Removing environment: {env_name}'):
                 environment.remove()

src/hatch/cli/project/metadata.py

@@ -19,7 +19,10 @@ def metadata(app, field):
         with app.project.location.as_cwd():
             environment = app.get_environment()
 
-            with app.status_waiting('Setting up build environment for missing dependencies') as status:
+            with app.status_waiting(
+                'Setting up build environment for missing dependencies',
+                condition=not environment.build_environment_exists(),
+            ) as status:
                 with environment.build_environment(app.project.metadata.build.requires):
                     status.stop()
 

src/hatch/cli/terminal.py

@@ -148,9 +148,11 @@ def display_table(self, title, columns, show_lines=False, column_options=None, f
         self.display(table)
 
     @contextmanager
-    def status_waiting(self, text='', final_text=None, **kwargs):
-        if not self.interactive or not self.console.is_terminal:
-            self.display_waiting(text)
+    def status_waiting(self, text='', final_text=None, condition=True, **kwargs):
+        if not condition or not self.interactive or not self.console.is_terminal:
+            if condition:
+                self.display_waiting(text)
+
             with MockStatus() as status:
                 yield status
         else:

src/hatch/cli/version/__init__.py

@@ -36,7 +36,10 @@ def version(app, desired_version):
         else:
             environment = app.get_environment()
 
-            with app.status_waiting('Setting up build environment for missing build dependencies') as status:
+            with app.status_waiting(
+                'Setting up build environment for missing build dependencies',
+                condition=not environment.build_environment_exists(),
+            ) as status:
                 with environment.build_environment(app.project.metadata.build.requires):
                     status.stop()
 

src/hatch/env/plugin/interface.py

@@ -598,6 +598,10 @@ def remove(self):
         This should perform the necessary steps to completely remove the environment from the system and will only
         be triggered manually by users with the [`env remove`](../../cli/reference.md#hatch-env-remove) or
         [`env prune`](../../cli/reference.md#hatch-env-prune) commands.
+
+        If the
+        [build environment](reference.md#hatch.env.plugin.interface.EnvironmentInterface.build_environment)
+        has a caching mechanism, this should remove that as well.
         """
 
     @abstractmethod
@@ -683,6 +687,14 @@ def get_build_process(self, build_environment, **kwargs):
         """
         return self.platform.capture_process(self.construct_build_command(**kwargs))
 
+    def build_environment_exists(self):
+        """
+        If the
+        [build environment](reference.md#hatch.env.plugin.interface.EnvironmentInterface.build_environment)
+        has a caching mechanism, this should indicate whether or not it has already been created.
+        """
+        return False
+
     def enter_shell(self, name, path, args):
         """
         Spawn a [shell](../../config/hatch.md#shell) within the environment.

src/hatch/env/virtual.py

@@ -6,7 +6,7 @@
 from hatch.env.plugin.interface import EnvironmentInterface
 from hatch.utils.fs import Path
 from hatch.utils.shells import ShellManager
-from hatch.venv.core import TempVirtualEnv, VirtualEnv
+from hatch.venv.core import VirtualEnv
 
 
 class VirtualEnvironment(EnvironmentInterface):
@@ -29,6 +29,9 @@ def __init__(self, *args, **kwargs):
             self.virtual_env_path = self.storage_path / directory
 
         self.virtual_env = VirtualEnv(self.virtual_env_path, self.platform, self.verbosity)
+        self.build_virtual_env = VirtualEnv(
+            self.virtual_env_path.parent / f'{self.virtual_env_path.name}-build', self.platform, self.verbosity
+        )
         self.shells = ShellManager(self)
 
         self._parent_python = None
@@ -51,6 +54,7 @@ def create(self):
 
     def remove(self):
         self.virtual_env.remove()
+        self.build_virtual_env.remove()
 
         # Clean up root directory of all virtual environments belonging to the project
         if self.storage_path.is_dir() and not any(self.storage_path.iterdir()):
@@ -86,11 +90,26 @@ def sync_dependencies(self):
 
     @contextmanager
     def build_environment(self, dependencies):
-        with self.get_env_vars(), TempVirtualEnv(self.parent_python, self.platform, self.verbosity):
-            self.platform.check_command(self.construct_pip_install_command(dependencies))
+        from packaging.requirements import Requirement
+
+        from hatchling.dep.core import dependencies_in_sync
+
+        if not self.build_environment_exists():
+            self.build_virtual_env.create(self.parent_python)
+
+        with self.get_env_vars(), self.build_virtual_env:
+            if not dependencies_in_sync(
+                [Requirement(d) for d in dependencies],
+                sys_path=self.build_virtual_env.sys_path,
+                environment=self.build_virtual_env.environment,
+            ):
+                self.platform.check_command(self.construct_pip_install_command(dependencies))
 
             yield
 
+    def build_environment_exists(self):
+        return self.build_virtual_env.exists()
+
     @contextmanager
     def command_context(self):
         with self.safe_activation():

tests/cli/build/test_build.py

@@ -1,5 +1,6 @@
 import pytest
 
+from hatch.config.constants import ConfigEnvVars
 from hatch.project.core import Project
 from hatchling.builders.constants import BuildEnvVars
 from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT
@@ -1000,14 +1001,35 @@ def test_shipped(hatch, temp_dir, helpers):
         result = hatch('new', project_name)
         assert result.exit_code == 0, result.output
 
-    path = temp_dir / 'my-app'
+    project_path = temp_dir / 'my-app'
+    data_path = temp_dir / 'data'
+    data_path.mkdir()
 
-    with path.as_cwd():
+    with project_path.as_cwd(env_vars={ConfigEnvVars.DATA: str(data_path)}):
         result = hatch('build')
 
     assert result.exit_code == 0, result.output
 
-    build_directory = path / 'dist'
+    env_data_path = data_path / 'env' / 'virtual'
+    assert env_data_path.is_dir()
+
+    project_data_path = env_data_path / project_path.name
+    assert project_data_path.is_dir()
+
+    storage_dirs = list(project_data_path.iterdir())
+    assert len(storage_dirs) == 1
+
+    storage_path = storage_dirs[0]
+    assert len(storage_path.name) == 8
+
+    env_dirs = list(storage_path.iterdir())
+    assert len(env_dirs) == 1
+
+    env_path = env_dirs[0]
+
+    assert env_path.name == f'{project_path.name}-build'
+
+    build_directory = project_path / 'dist'
     assert build_directory.is_dir()
 
     artifacts = list(build_directory.iterdir())
@@ -1020,14 +1042,26 @@ def test_shipped(hatch, temp_dir, helpers):
         f"""
         Setting up build environment
         [sdist]
-        {sdist_path.relative_to(path)}
+        {sdist_path.relative_to(project_path)}
 
-        Setting up build environment
         [wheel]
-        {wheel_path.relative_to(path)}
+        {wheel_path.relative_to(project_path)}
         """
     )
 
+    # Test removal while we're here
+    with project_path.as_cwd(env_vars={ConfigEnvVars.DATA: str(data_path)}):
+        result = hatch('env', 'remove')
+
+    assert result.exit_code == 0, result.output
+    assert result.output == helpers.dedent(
+        """
+        Removing environment: default
+        """
+    )
+
+    assert not storage_path.is_dir()
+
 
 @pytest.mark.allow_backend_process
 @pytest.mark.requires_internet
@@ -1038,9 +1072,11 @@ def test_build_dependencies(hatch, temp_dir, helpers):
         result = hatch('new', project_name)
         assert result.exit_code == 0, result.output
 
-    path = temp_dir / 'my-app'
+    project_path = temp_dir / 'my-app'
+    data_path = temp_dir / 'data'
+    data_path.mkdir()
 
-    build_script = path / DEFAULT_BUILD_SCRIPT
+    build_script = project_path / DEFAULT_BUILD_SCRIPT
     build_script.write_text(
         helpers.dedent(
             """
@@ -1060,25 +1096,25 @@ def build(self, *args, **kwargs):
         )
     )
 
-    project = Project(path)
+    project = Project(project_path)
     config = dict(project.raw_config)
     config['tool']['hatch']['build'] = {
         'targets': {'custom': {'dependencies': ['binary'], 'path': DEFAULT_BUILD_SCRIPT}},
     }
     project.save_config(config)
 
-    with path.as_cwd():
+    with project_path.as_cwd(env_vars={ConfigEnvVars.DATA: str(data_path)}):
         result = hatch('build', '-t', 'custom')
 
     assert result.exit_code == 0, result.output
 
-    build_directory = path / 'dist'
+    build_directory = project_path / 'dist'
     assert build_directory.is_dir()
 
     artifacts = list(build_directory.iterdir())
     assert len(artifacts) == 1
 
-    output_file = path / 'test.txt'
+    output_file = project_path / 'test.txt'
     assert output_file.is_file()
 
     assert str(output_file.read_text()) == "(1.0, 'KiB')"
@@ -1089,6 +1125,6 @@ def build(self, *args, **kwargs):
         f"""
         Setting up build environment
         [custom]
-        {wheel_path.relative_to(path)}
+        {wheel_path.relative_to(project_path)}
         """
     )