Add option to skip installing directory dependencies (python-poetry#6845

)

docs/cli.md

@@ -225,6 +225,14 @@ If you want to skip this installation, use the `--no-root` option.
 poetry install --no-root
 ```
 
+Similar to `--no-root` you can use `--no-directory` to skip directory path dependencies:
+
+```bash
+poetry install --no-directory
+```
+
+This is mainly useful for caching in CI or when building Docker images. See the [FAQ entry]({{< relref "faq#poetry-busts-my-docker-cache-because-it-requires-me-to-copy-my-source-files-in-before-installing-3rd-party-dependencies" >}}) for more information on this option.
+
 By default `poetry` does not compile Python source files to bytecode during installation.
 This speeds up the installation process, but the first execution may take a little more
 time because Python then compiles source files to bytecode automatically.
@@ -240,6 +248,7 @@ The `--compile` option has no effect if `installer.modern-installation`
 is set to `false` because the old installer always compiles source files to bytecode.
 {{% /note %}}
 
+
 ### Options
 
 * `--without`: The dependency groups to ignore.
@@ -248,6 +257,7 @@ is set to `false` because the old installer always compiles source files to byte
 * `--only-root`: Install only the root project, exclude all dependencies.
 * `--sync`: Synchronize the environment with the locked packages and the specified groups.
 * `--no-root`: Do not install the root package (your project).
+* `--no-directory`: Skip all directory path dependencies (including transitive ones).
 * `--dry-run`: Output the operations but do not execute anything (implicitly enables --verbose).
 * `--extras (-E)`: Features to install (multiple values allowed).
 * `--all-extras`: Install all extra features (conflicts with --extras).

docs/faq.md

@@ -189,3 +189,37 @@ This is done so to be compliant with the broader Python ecosystem.
 
 For example, if Poetry builds a distribution for a project that uses a version that is not valid according to
 [PEP 440](https://peps.python.org/pep-0440), third party tools will be unable to parse the version correctly.
+
+
+### Poetry busts my Docker cache because it requires me to COPY my source files in before installing 3rd party dependencies
+
+By default running `poetry install ...` requires you to have your source files present (both the "root" package and any directory path dependencies you might have).
+This interacts poorly with Docker's caching mechanisms because any change to a source file will make any layers (subsequent commands in your Dockerfile) re-run.
+For example, you might have a Dockerfile that looks something like this:
+
+```text
+FROM python
+COPY pyproject.toml poetry.lock .
+COPY src/ ./src
+RUN pip install poetry && poetry install --no-dev
+```
+
+As soon as *any* source file changes, the cache for the `RUN` layer will be invalidated, which forces all 3rd party dependencies (likely the slowest step out of these) to be installed again if you changed any files in `src/`.
+
+To avoid this cache busting you can split this into two steps:
+
+1. Install 3rd party dependencies.
+2. Copy over your source code and install just the source code.
+
+This might look something like this:
+
+```text
+FROM python
+COPY pyproject.toml poetry.lock .
+RUN pip install poetry && poetry install --no-root --no-directory
+COPY src/ ./src
+RUN poetry install --no-dev
+```
+
+The two key options we are using here are `--no-root` (skips installing the project source) and `--no-directory` (skips installing any local directory path dependencies, you can omit this if you don't have any).
+[More information on the options available for `poetry install`]({{< relref "cli#install" >}}).

src/poetry/console/commands/install.py

@@ -30,6 +30,16 @@ class InstallCommand(InstallerCommand):
         option(
             "no-root", None, "Do not install the root package (the current project)."
         ),
+        option(
+            "no-directory",
+            None,
+            (
+                "Do not install any directory path dependencies; useful to install"
+                " dependencies without source code, e.g. for caching of Docker layers)"
+            ),
+            flag=True,
+            multiple=False,
+        ),
         option(
             "dry-run",
             None,
@@ -148,6 +158,7 @@ def handle(self) -> int:
             with_synchronization = True
 
         self.installer.only_groups(self.activated_groups)
+        self.installer.skip_directory(self.option("no-directory"))
         self.installer.dry_run(self.option("dry-run"))
         self.installer.requires_synchronization(with_synchronization)
         self.installer.executor.enable_bytecode_compilation(self.option("compile"))

src/poetry/installation/installer.py

@@ -59,6 +59,7 @@ def __init__(
         self._verbose = False
         self._write_lock = True
         self._groups: Iterable[str] | None = None
+        self._skip_directory = False
 
         self._execute_operations = True
         self._lock = False
@@ -150,6 +151,11 @@ def update(self, update: bool = True) -> Installer:
 
         return self
 
+    def skip_directory(self, skip_directory: bool = False) -> Installer:
+        self._skip_directory = skip_directory
+
+        return self
+
     def lock(self, update: bool = True) -> Installer:
         """
         Prepare the installer for locking only.
@@ -334,6 +340,7 @@ def _do_install(self) -> int:
             ops = solver.solve(use_latest=self._whitelist).calculate_operations(
                 with_uninstalls=self._requires_synchronization,
                 synchronize=self._requires_synchronization,
+                skip_directory=self._skip_directory,
             )
 
         if not self._requires_synchronization:

src/poetry/puzzle/transaction.py

@@ -27,7 +27,11 @@ def __init__(
         self._root_package = root_package
 
     def calculate_operations(
-        self, with_uninstalls: bool = True, synchronize: bool = False
+        self,
+        with_uninstalls: bool = True,
+        synchronize: bool = False,
+        *,
+        skip_directory: bool = False,
     ) -> list[Operation]:
         from poetry.installation.operations import Install
         from poetry.installation.operations import Uninstall
@@ -70,7 +74,10 @@ def calculate_operations(
 
                     break
 
-            if not installed:
+            if not (
+                installed
+                or (skip_directory and result_package.source_type == "directory")
+            ):
                 operations.append(Install(result_package, priority=priority))
 
         if with_uninstalls:

tests/console/commands/test_install.py

@@ -184,6 +184,24 @@ def test_compile_option_is_passed_to_the_installer(
     enable_bytecode_compilation_mock.assert_called_once_with(compile)
 
 
+@pytest.mark.parametrize("skip_directory_cli_value", [True, False])
+def test_no_directory_is_passed_to_installer(
+    tester: CommandTester, mocker: MockerFixture, skip_directory_cli_value: bool
+):
+    """
+    The --no-directory option is passed to the installer.
+    """
+
+    mocker.patch.object(tester.command.installer, "run", return_value=1)
+
+    if skip_directory_cli_value is True:
+        tester.execute("--no-directory")
+    else:
+        tester.execute()
+
+    assert tester.command.installer._skip_directory is skip_directory_cli_value
+
+
 def test_no_all_extras_doesnt_populate_installer(
     tester: CommandTester, mocker: MockerFixture
 ):

tests/installation/test_installer.py

@@ -60,12 +60,12 @@ class Executor(BaseExecutor):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
-        self._installs: list[DependencyPackage] = []
+        self._installs: list[Package] = []
         self._updates: list[DependencyPackage] = []
         self._uninstalls: list[DependencyPackage] = []
 
     @property
-    def installations(self) -> list[DependencyPackage]:
+    def installations(self) -> list[Package]:
         return self._installs
 
     @property
@@ -1276,14 +1276,18 @@ def test_run_installs_with_local_poetry_directory_and_extras(
     assert installer.executor.installations_count == 2
 
 
-def test_run_installs_with_local_poetry_directory_transitive(
+@pytest.mark.parametrize("skip_directory", [True, False])
+def test_run_installs_with_local_poetry_directory_and_skip_directory_flag(
     installer: Installer,
     locker: Locker,
     repo: Repository,
     package: ProjectPackage,
-    tmpdir: Path,
     fixture_dir: FixtureDirGetter,
+    skip_directory: bool,
 ):
+    """When we set Installer.skip_directory(True) no path dependencies should
+    be installed (including transitive dependencies).
+    """
     root_dir = fixture_dir("directory")
     package.root_dir = root_dir
     locker.set_lock_path(root_dir)
@@ -1299,14 +1303,27 @@ def test_run_installs_with_local_poetry_directory_transitive(
     repo.add_package(get_package("pendulum", "1.4.4"))
     repo.add_package(get_package("cachy", "0.2.0"))
 
+    installer.skip_directory(skip_directory)
+
     result = installer.run()
     assert result == 0
 
+    executor: Executor = installer.executor  # type: ignore
+
     expected = fixture("with-directory-dependency-poetry-transitive")
 
     assert locker.written_data == expected
 
-    assert installer.executor.installations_count == 6
+    directory_installs = [
+        p.name for p in executor.installations if p.source_type == "directory"
+    ]
+
+    if skip_directory:
+        assert not directory_installs, directory_installs
+        assert installer.executor.installations_count == 2
+    else:
+        assert directory_installs, directory_installs
+        assert installer.executor.installations_count == 6
 
 
 def test_run_installs_with_local_poetry_file_transitive(