Backed out 10 changesets (bug 1712151, bug 1724279, bug 1730712, bug …

…1717051, bug 1723031, bug 1731145) for causing failures on test_yaml.py

Backed out changeset 7f64d538701b (bug 1723031)
Backed out changeset 394152994966 (bug 1723031)
Backed out changeset 9bfeb01bcc9a (bug 1723031)
Backed out changeset 3d283616a57d (bug 1730712)
Backed out changeset bc677b409650 (bug 1724279)
Backed out changeset 784c94c2f528 (bug 1723031)
Backed out changeset 6e1bde40e3b4 (bug 1723031)
Backed out changeset 7adf7e2136a3 (bug 1712151)
Backed out changeset 2aef162b9a1b (bug 1717051)
Backed out changeset 9beeb6d3d95b (bug 1731145)

.gitignore

@@ -14,16 +14,11 @@ ID
 !id/
 .DS_Store*
 *.pdb
+*.egg-info
 .eslintcache
 # Filesystem temporaries
 .fuse_hidden*
 
-# Ignore Python .egg-info directories for first-party modules (but,
-# still add vendored packages' .egg-info directories)
-*.egg-info
-!third_party/python/**/*.egg-info
-!testing/web-platform/tests/tools/third_party/**/*.egg-info
-
 # Vim swap files.
 .*.sw[a-z]
 .sw[a-z]

.hgignore

@@ -7,16 +7,13 @@
 (^|/)ID$
 (^|/)\.DS_Store$
 \.pdb
+\.egg-info
 \.eslintcache
 \.gcda
 \.gcno
 \.gcov
 compile_commands\.json
 
-# Ignore Python .egg-info directories for first-party modules (but,
-# still add vendored packages' .egg-info directories)
-^(?=.*\.egg-info/)(?!^third_party/python/)(?!^testing/web-platform/tests/tools/third_party/)
-
 # Vim swap files.
 ^\.sw.$
 .[^/]*\.sw.$

build/build_virtualenv_packages.txt

@@ -1,2 +1,2 @@
 packages.txt:build/common_virtualenv_packages.txt
-vendored:third_party/python/glean_parser
+pth:third_party/python/glean_parser

build/common_virtualenv_packages.txt

@@ -44,82 +44,82 @@ pth:testing/mozbase/mozversion
 pth:testing/raptor
 pth:testing/talos
 pth:testing/web-platform
-vendored:testing/web-platform/tests/tools/third_party/h2
-vendored:testing/web-platform/tests/tools/third_party/hpack
-vendored:testing/web-platform/tests/tools/third_party/html5lib
-vendored:testing/web-platform/tests/tools/third_party/hyperframe
-vendored:testing/web-platform/tests/tools/third_party/pywebsocket3
-vendored:testing/web-platform/tests/tools/third_party/webencodings
-vendored:testing/web-platform/tests/tools/wptserve
-vendored:testing/web-platform/tests/tools/wptrunner
+pth:testing/web-platform/tests/tools/third_party/certifi
+pth:testing/web-platform/tests/tools/third_party/h2
+pth:testing/web-platform/tests/tools/third_party/hpack
+pth:testing/web-platform/tests/tools/third_party/html5lib
+pth:testing/web-platform/tests/tools/third_party/hyperframe
+pth:testing/web-platform/tests/tools/third_party/pywebsocket3
+pth:testing/web-platform/tests/tools/third_party/webencodings
+pth:testing/web-platform/tests/tools/wptserve
+pth:testing/web-platform/tests/tools/wptrunner
 pth:testing/xpcshell
-vendored:third_party/python/aiohttp
-vendored:third_party/python/appdirs
-vendored:third_party/python/async_timeout
-vendored:third_party/python/atomicwrites
-vendored:third_party/python/attrs
-vendored:third_party/python/blessings
-vendored:third_party/python/cbor2
-vendored:third_party/python/certifi
-vendored:third_party/python/chardet
-vendored:third_party/python/Click
-vendored:third_party/python/compare_locales
-vendored:third_party/python/cookies
-vendored:third_party/python/cram
-vendored:third_party/python/diskcache
-vendored:third_party/python/distro
-vendored:third_party/python/dlmanager
-vendored:third_party/python/ecdsa
-vendored:third_party/python/esprima
-vendored:third_party/python/fluent.migrate
-vendored:third_party/python/fluent.syntax
-vendored:third_party/python/funcsigs
-vendored:third_party/python/gyp/pylib
-vendored:third_party/python/idna
-vendored:third_party/python/idna-ssl
-vendored:third_party/python/importlib_metadata
-vendored:third_party/python/iso8601
-vendored:third_party/python/Jinja2
-vendored:third_party/python/jsmin
-vendored:third_party/python/json-e
-vendored:third_party/python/jsonschema
-vendored:third_party/python/MarkupSafe/src
-vendored:third_party/python/mohawk
-vendored:third_party/python/more_itertools
-vendored:third_party/python/mozilla_version
-vendored:third_party/python/multidict
-vendored:third_party/python/packaging
-vendored:third_party/python/pathspec
-vendored:third_party/python/pip_tools
-vendored:third_party/python/pluggy
-vendored:third_party/python/ply
-vendored:third_party/python/py
-vendored:third_party/python/pyasn1
-vendored:third_party/python/pyasn1_modules
-vendored:third_party/python/pylru
-vendored:third_party/python/pyparsing
-vendored:third_party/python/pyrsistent
-vendored:third_party/python/pystache
-vendored:third_party/python/pytest
-vendored:third_party/python/python-hglib
-vendored:third_party/python/pytoml
-vendored:third_party/python/PyYAML/lib3/
-vendored:third_party/python/redo
-vendored:third_party/python/requests
-vendored:third_party/python/requests_unixsocket
-vendored:third_party/python/responses
-vendored:third_party/python/rsa
-vendored:third_party/python/sentry_sdk
-vendored:third_party/python/six
-vendored:third_party/python/slugid
-vendored:third_party/python/taskcluster
-vendored:third_party/python/taskcluster_urls
-vendored:third_party/python/typing_extensions
-vendored:third_party/python/urllib3
-vendored:third_party/python/voluptuous
-vendored:third_party/python/yamllint
-vendored:third_party/python/yarl
-vendored:third_party/python/zipp
+pth:third_party/python/aiohttp
+pth:third_party/python/appdirs
+pth:third_party/python/async_timeout
+pth:third_party/python/atomicwrites
+pth:third_party/python/attrs
+pth:third_party/python/blessings
+pth:third_party/python/cbor2
+pth:third_party/python/chardet
+pth:third_party/python/Click
+pth:third_party/python/compare_locales
+pth:third_party/python/cookies
+pth:third_party/python/cram
+pth:third_party/python/diskcache
+pth:third_party/python/distro
+pth:third_party/python/dlmanager
+pth:third_party/python/ecdsa
+pth:third_party/python/esprima
+pth:third_party/python/fluent.migrate
+pth:third_party/python/fluent.syntax
+pth:third_party/python/funcsigs
+pth:third_party/python/gyp/pylib
+pth:third_party/python/idna
+pth:third_party/python/idna-ssl
+pth:third_party/python/importlib_metadata
+pth:third_party/python/iso8601
+pth:third_party/python/Jinja2
+pth:third_party/python/jsmin
+pth:third_party/python/json-e
+pth:third_party/python/jsonschema
+pth:third_party/python/MarkupSafe/src
+pth:third_party/python/mohawk
+pth:third_party/python/more_itertools
+pth:third_party/python/mozilla_version
+pth:third_party/python/multidict
+pth:third_party/python/packaging
+pth:third_party/python/pathspec
+pth:third_party/python/pip_tools
+pth:third_party/python/pluggy
+pth:third_party/python/ply
+pth:third_party/python/py
+pth:third_party/python/pyasn1
+pth:third_party/python/pyasn1_modules
+pth:third_party/python/pylru
+pth:third_party/python/pyparsing
+pth:third_party/python/pyrsistent
+pth:third_party/python/pystache
+pth:third_party/python/pytest
+pth:third_party/python/python-hglib
+pth:third_party/python/pytoml
+pth:third_party/python/PyYAML/lib3/
+pth:third_party/python/redo
+pth:third_party/python/requests
+pth:third_party/python/requests_unixsocket
+pth:third_party/python/responses
+pth:third_party/python/rsa
+pth:third_party/python/sentry_sdk
+pth:third_party/python/six
+pth:third_party/python/slugid
+pth:third_party/python/taskcluster
+pth:third_party/python/taskcluster_urls
+pth:third_party/python/typing_extensions
+pth:third_party/python/urllib3
+pth:third_party/python/voluptuous
+pth:third_party/python/yamllint
+pth:third_party/python/yarl
+pth:third_party/python/zipp
 pth:toolkit/components/telemetry/tests/marionette/harness
 pth:tools
 pth:tools/moztreedocs

build/mach_initialize.py

@@ -9,7 +9,6 @@
 import platform
 import shutil
 import site
-import subprocess
 import sys
 
 if sys.version_info[0] < 3:
@@ -142,6 +141,35 @@ class MetaPathFinder(object):
     },
 }
 
+
+def search_path(mozilla_dir, packages_txt):
+    with open(os.path.join(mozilla_dir, packages_txt)) as f:
+        packages = [
+            line.strip().split(":", maxsplit=1)
+            for line in f
+            if not line.lstrip().startswith("#")
+        ]
+
+    def handle_package(action, package):
+        if action == "packages.txt":
+            for p in search_path(mozilla_dir, package):
+                yield os.path.join(mozilla_dir, p)
+
+        if action == "pth":
+            yield os.path.join(mozilla_dir, package)
+
+    for current_action, current_package in packages:
+        for path in handle_package(current_action, current_package):
+            yield path
+
+
+def mach_sys_path(mozilla_dir):
+    return [
+        os.path.join(mozilla_dir, path)
+        for path in search_path(mozilla_dir, "build/mach_virtualenv_packages.txt")
+    ]
+
+
 INSTALL_PYTHON_GUIDANCE_LINUX = """
 See https://firefox-source-docs.mozilla.org/setup/linux_build.html#installingpython
 for guidance on how to install Python on your system.
@@ -168,102 +196,6 @@ class MetaPathFinder(object):
 """.strip()
 
 
-def _scrub_system_site_packages():
-    site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
-    sys.path = [path for path in sys.path if path not in site_paths]
-
-
-def _activate_python_environment(topsrcdir):
-    # We need the "mach" module to access the logic to parse virtualenv
-    # requirements. Since that depends on "packaging" (and, transitively,
-    # "pyparsing"), we add those to the path too.
-    sys.path[0:0] = [
-        os.path.join(topsrcdir, module)
-        for module in (
-            os.path.join("python", "mach"),
-            os.path.join("third_party", "python", "packaging"),
-            os.path.join("third_party", "python", "pyparsing"),
-        )
-    ]
-
-    from mach.requirements import MachEnvRequirements
-
-    thunderbird_dir = os.path.join(topsrcdir, "comm")
-    is_thunderbird = os.path.exists(thunderbird_dir) and bool(
-        os.listdir(thunderbird_dir)
-    )
-
-    requirements = MachEnvRequirements.from_requirements_definition(
-        topsrcdir,
-        is_thunderbird,
-        True,
-        os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
-    )
-
-    if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
-        env_var = (
-            "MOZ_AUTOMATION"
-            if os.environ.get("MOZ_AUTOMATION")
-            else "MACH_USE_SYSTEM_PYTHON"
-        )
-
-        has_pip = (
-            subprocess.run(
-                [sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL
-            ).returncode
-            == 0
-        )
-        # There are environments in CI that aren't prepared to provide any Mach dependency
-        # packages. Changing this is a nontrivial endeavour, so guard against having
-        # non-optional Mach requirements.
-        assert (
-            not requirements.pypi_requirements
-        ), "Mach pip package requirements must be optional."
-        if has_pip:
-            pip = [sys.executable, "-m", "pip"]
-            check_result = subprocess.run(
-                pip + ["check"],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                universal_newlines=True,
-            )
-            if check_result.returncode:
-                print(check_result.stdout, file=sys.stderr)
-                subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
-                raise Exception(
-                    'According to "pip check", the current Python '
-                    "environment has package-compatibility issues."
-                )
-
-            package_result = requirements.validate_environment_packages(pip)
-            if not package_result.has_all_packages:
-                print(
-                    "Skipping automatic management of Python dependencies since "
-                    f"the '{env_var}' environment variable is set.\n"
-                    "The following issues were found while validating your Python "
-                    "environment:"
-                )
-                print(package_result.report())
-                sys.exit(1)
-        else:
-            # Pip isn't installed to the system Python environment, so we can't use
-            # it to verify compatibility with Mach. Remove the system site-packages
-            # from the import scope so that Mach behaves as though all of its
-            # (optional) dependencies are not installed.
-            _scrub_system_site_packages()
-
-    elif sys.prefix == sys.base_prefix:
-        # We're in an environment where we normally use the Mach virtualenv,
-        # but we're running a "nativecmd" such as "create-mach-environment".
-        # Remove global site packages from sys.path to improve isolation accordingly.
-        _scrub_system_site_packages()
-
-    sys.path[0:0] = [
-        os.path.join(topsrcdir, pth.path)
-        for pth in requirements.pth_requirements + requirements.vendored_requirements
-    ]
-
-
 def initialize(topsrcdir):
     # Ensure we are running Python 3.6+. We run this check as soon as
     # possible to avoid a cryptic import/usage error.
@@ -287,9 +219,15 @@ def initialize(topsrcdir):
     if os.path.exists(deleted_dir):
         shutil.rmtree(deleted_dir, ignore_errors=True)
 
+    if sys.prefix == sys.base_prefix:
+        # We are not in a virtualenv. Remove global site packages
+        # from sys.path.
+        site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
+        sys.path = [path for path in sys.path if path not in site_paths]
+
     state_dir = _create_state_dir()
-    _activate_python_environment(topsrcdir)
 
+    sys.path[0:0] = mach_sys_path(topsrcdir)
     import mach.base
     import mach.main
     from mach.util import setenv

build/mach_virtualenv_packages.txt

@@ -3,7 +3,5 @@ packages.txt:build/common_virtualenv_packages.txt
 # and it has to be built from source.
 pypi-optional:glean-sdk==40.0.0:telemetry will not be collected
 # Mach gracefully handles the case where `psutil` is unavailable.
-# We aren't (yet) able to pin packages in automation, so we have to
-# support down to the oldest locally-installed version (5.4.2).
-pypi-optional:psutil>=5.4.2,<=5.8.0:telemetry will be missing some data
-pypi-optional:zstandard>=0.11.1,<=0.15.2:zstd archives will not be possible to extract
+pypi-optional:psutil==5.8.0:telemetry will be missing some data
+pypi:zstandard==0.15.2

build/python-test_virtualenv_packages.txt

@@ -1,3 +1,2 @@
 packages.txt:build/common_virtualenv_packages.txt
-vendored:third_party/python/glean_parser
-
+pth:third_party/python/glean_parser

configure.py

@@ -21,11 +21,8 @@
 
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
-sys.path.insert(0, os.path.join(base_dir, "python", "mach"))
 sys.path.insert(0, os.path.join(base_dir, "python", "mozboot"))
 sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
-sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
-sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
 from mozbuild.configure import (
     ConfigureSandbox,