Bug 1730712: Split site manager into command and mach managers r=ahal

The command site manager needs to be able to do ad-hoc pip
installations, while the Mach site manager needs to manage
the system `sys.path` and conditionally create an on-disk
virtualenv.

By splitting the class into two, we can now give each use case the
attention it deserves.

Differential Revision: https://phabricator.services.mozilla.com/D129529

build/mach_initialize.py

@@ -8,8 +8,6 @@
 import os
 import platform
 import shutil
-import site
-import subprocess
 import sys
 
 if sys.version_info[0] < 3:
@@ -168,14 +166,9 @@ class MetaPathFinder(object):
 """.strip()
 
 
-def _scrub_system_site_packages():
-    site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
-    sys.path = [path for path in sys.path if path not in site_paths]
-
-
 def _activate_python_environment(topsrcdir, state_dir):
-    # We need the "mach" module to access the logic to parse site
-    # requirements. Since that depends on "packaging" (and, transitively,
+    # We need the "mach" module to access the logic to activate the top-level
+    # Mach site. Since that depends on "packaging" (and, transitively,
     # "pyparsing"), we add those to the path too.
     sys.path[0:0] = [
         os.path.join(topsrcdir, module)
@@ -187,98 +180,24 @@ def _activate_python_environment(topsrcdir, state_dir):
     ]
 
     from mach.site import (
-        MozSiteMetadata,
+        MachSiteManager,
+        VirtualenvOutOfDateException,
         MozSiteMetadataOutOfDateError,
-        MozSiteManager,
     )
 
     try:
-        mach_site = MozSiteManager(
+        mach_environment = MachSiteManager.from_environment(
             topsrcdir,
-            os.path.join(state_dir, "_virtualenvs"),
-            "mach",
-        )
-        active_site_metadata = MozSiteMetadata.from_runtime()
-        is_mach_site = active_site_metadata and active_site_metadata.site_name == "mach"
-    except MozSiteMetadataOutOfDateError as e:
-        print(e)
-        print('This should be resolved by running "./mach create-mach-environment".')
-        sys.exit(1)
-
-    requirements = mach_site.requirements()
-    if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
-        env_var = (
-            "MOZ_AUTOMATION"
-            if os.environ.get("MOZ_AUTOMATION")
-            else "MACH_USE_SYSTEM_PYTHON"
+            # normpath state_dir to normalize msys-style slashes.
+            os.path.normpath(state_dir),
         )
-
-        has_pip = (
-            subprocess.run(
-                [sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL
-            ).returncode
-            == 0
+        mach_environment.activate()
+    except (VirtualenvOutOfDateException, MozSiteMetadataOutOfDateError):
+        print(
+            'The "mach" virtualenv is not up-to-date, please run '
+            '"./mach create-mach-environment"'
         )
-        # There are environments in CI that aren't prepared to provide any Mach dependency
-        # packages. Changing this is a nontrivial endeavour, so guard against having
-        # non-optional Mach requirements.
-        assert (
-            not requirements.pypi_requirements
-        ), "Mach pip package requirements must be optional."
-        if has_pip:
-            pip = [sys.executable, "-m", "pip"]
-            check_result = subprocess.run(
-                pip + ["check"],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                universal_newlines=True,
-            )
-            if check_result.returncode:
-                print(check_result.stdout, file=sys.stderr)
-                subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
-                raise Exception(
-                    'According to "pip check", the current Python '
-                    "environment has package-compatibility issues."
-                )
-
-            package_result = requirements.validate_environment_packages(pip)
-            if not package_result.has_all_packages:
-                print(
-                    "Skipping automatic management of Python dependencies since "
-                    f"the '{env_var}' environment variable is set.\n"
-                    "The following issues were found while validating your Python "
-                    "environment:"
-                )
-                print(package_result.report())
-                sys.exit(1)
-        else:
-            # Pip isn't installed to the system Python environment, so we can't use
-            # it to verify compatibility with Mach. Remove the system site-packages
-            # from the import scope so that Mach behaves as though all of its
-            # (optional) dependencies are not installed.
-            _scrub_system_site_packages()
-
-        sys.path[0:0] = requirements.pths_as_absolute(topsrcdir)
-    elif is_mach_site:
-        # We're running in the Mach virtualenv - check that it's up-to-date.
-        # Note that the "pip package check" exists to ensure that a virtualenv isn't
-        # corrupted by ad-hoc pip installs. Since the Mach virtualenv is unlikely
-        # to be affected by such installs, and since it takes ~400ms to get the list
-        # of installed pip packages (a *lot* of time to wait during Mach init), we
-        # skip verifying that our pip packages exist.
-        if not mach_site.up_to_date():
-            print(
-                'The "mach" virtualenv is not up-to-date, please run '
-                '"./mach create-mach-environment"'
-            )
-            sys.exit(1)
-    else:
-        # We're in an environment where we normally *would* use the Mach virtualenv,
-        # but we're running a "nativecmd" such as "create-mach-environment".
-        # Remove global site packages from sys.path to improve isolation accordingly.
-        _scrub_system_site_packages()
-
-        sys.path[0:0] = requirements.pths_as_absolute(topsrcdir)
+        sys.exit(1)
 
 
 def initialize(topsrcdir):

configure.py

@@ -22,7 +22,7 @@
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
-from mach.site import MozSiteManager
+from mach.site import CommandSiteManager
 from mozbuild.configure import (
     ConfigureSandbox,
     TRACE,
@@ -232,14 +232,13 @@ def _activate_build_virtualenv():
     topobjdir = os.path.realpath(".")
     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 
-    build_site = MozSiteManager(
+    build_site = CommandSiteManager(
         topsrcdir,
         os.path.join(topobjdir, "_virtualenvs"),
         "build",
     )
-    if not build_site.up_to_date():
-        print("Creating Python 3 virtualenv")
-        build_site.build()
+    if not build_site.ensure():
+        print("Created Python 3 virtualenv")
     build_site.activate()