Bug 1676533 - Consult the base revision for file hashes instead of th…

…e on-disk files in `mach artifact` r=ahal

This enables `mach artifact` and `mach bootstrap` to not fail due to local changes.

Differential Revision: https://phabricator.services.mozilla.com/D96892

python/mozversioncontrol/mozversioncontrol/__init__.py

@@ -109,6 +109,7 @@ def __exit__(self, exc_type, exc_value, exc_tb):
         pass
 
     def _run(self, *args, **runargs):
+        universal_newlines = runargs.get("universal_newlines", True)
         return_codes = runargs.get("return_codes", [])
 
         cmd = (self._tool,) + args
@@ -117,7 +118,7 @@ def _run(self, *args, **runargs):
                 cmd,
                 cwd=self.path,
                 env=ensure_subprocess_env(self._env),
-                universal_newlines=True,
+                universal_newlines=universal_newlines,
             )
         except subprocess.CalledProcessError as e:
             if e.returncode in return_codes:
@@ -228,6 +229,12 @@ def get_tracked_files_finder(self):
         example, commits to the repo during the Finder's lifetime.
         """
 
+    @abc.abstractmethod
+    def get_file_content(self, path, revision=None):
+        """Return as a bytestring the contents of the file as of the given
+        revision, or the current revision if none is provided.
+        """
+
     @abc.abstractmethod
     def working_directory_clean(self, untracked=False, ignored=False):
         """Determine if the working directory is free of modifications.
@@ -332,12 +339,16 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self._client.close()
 
     def _run(self, *args, **runargs):
+        universal_newlines = runargs.get("universal_newlines", True)
         if not self._client.server:
             return super(HgRepository, self)._run(*args, **runargs)
 
         # hglib requires bytes on python 3
         args = [a.encode("utf-8") if not isinstance(a, bytes) else a for a in args]
-        return self._client.rawcommand(args).decode("utf-8")
+        res = self._client.rawcommand(args)
+        if universal_newlines:
+            return res.decode("utf-8")
+        return res
 
     def get_commit_time(self):
         newest_public_revision_time = self._run(
@@ -456,6 +467,12 @@ def get_tracked_files_finder(self):
         )
         return FileListFinder(files)
 
+    def get_file_content(self, path, revision=None):
+        args = ["cat", path]
+        if revision:
+            args += ["-r", revision]
+        return self._run(*args, universal_newlines=False)
+
     def working_directory_clean(self, untracked=False, ignored=False):
         args = ["status", "--modified", "--added", "--removed", "--deleted"]
         if untracked:
@@ -595,6 +612,10 @@ def get_tracked_files_finder(self):
         files = [p for p in self._run("ls-files", "-z").split("\0") if p]
         return FileListFinder(files)
 
+    def get_file_content(self, path, revision=None):
+        revision = revision or "HEAD"
+        return self._run("show", revision + ":" + path, universal_newlines=False)
+
     def working_directory_clean(self, untracked=False, ignored=False):
         args = ["status", "--porcelain"]
 

python/mozversioncontrol/test/python.ini

@@ -2,6 +2,7 @@
 subsuite=mozversioncontrol
 
 [test_context_manager.py]
+[test_file_content.py]
 [test_push_to_try.py]
 [test_workdir_outgoing.py]
 [test_working_directory.py]

python/mozversioncontrol/test/test_file_content.py

@@ -0,0 +1,42 @@
+from __future__ import absolute_import
+
+import mozunit
+
+from mozversioncontrol import get_repository_object
+
+STEPS = {
+    "hg": [
+        """
+    echo "foo" > bar
+    """,
+        """
+    hg commit -m "Updated bar"
+    """,
+    ],
+    "git": [
+        """
+    echo "foo" > bar
+    """,
+        """
+    git commit -am "Updated bar"
+    """,
+    ],
+}
+
+
+def test_file_content(repo):
+    vcs = get_repository_object(repo.strpath)
+    head_ref = vcs.head_ref
+    assert vcs.get_file_content("foo") == b"foo\n"
+    assert vcs.get_file_content("bar") == b"bar\n"
+    next(repo.step)
+    assert vcs.get_file_content("foo") == b"foo\n"
+    assert vcs.get_file_content("bar") == b"bar\n"
+    next(repo.step)
+    assert vcs.get_file_content("foo") == b"foo\n"
+    assert vcs.get_file_content("bar") == b"foo\n"
+    assert vcs.get_file_content("bar", revision=head_ref) == b"bar\n"
+
+
+if __name__ == "__main__":
+    mozunit.main()

taskcluster/taskgraph/util/hash.py

@@ -21,9 +21,34 @@ def hash_path(path):
         return hashlib.sha256(fh.read()).hexdigest()
 
 
+@memoize
+def hash_path_as_of_base_revision(base_path, path):
+    repo = get_repository(base_path)
+    base_revision = get_base_revision(base_path)
+    return hashlib.sha256(
+        repo.get_file_content(path, revision=base_revision)
+    ).hexdigest()
+
+
+@memoize
+def get_repository(base_path):
+    return get_repository_object(base_path)
+
+
 @memoize
 def get_file_finder(base_path):
-    return get_repository_object(base_path).get_tracked_files_finder()
+    return get_repository(base_path).get_tracked_files_finder()
+
+
+@memoize
+def get_dirty(base_path):
+    repo = get_repository(base_path)
+    return set(repo.get_outgoing_files()) | set(repo.get_changed_files())
+
+
+@memoize
+def get_base_revision(base_path):
+    return get_repository(base_path).base_ref
 
 
 def hash_paths(base_path, patterns):
@@ -35,8 +60,9 @@ def hash_paths(base_path, patterns):
     Each file is hashed. The list of all hashes and file paths is then
     itself hashed to produce the result.
     """
-    finder = get_file_finder(base_path)
+    dirty = get_dirty(base_path)
     h = hashlib.sha256()
+    finder = get_file_finder(base_path)
     files = {}
     for pattern in patterns:
         found = list(finder.find(pattern))
@@ -45,14 +71,12 @@ def hash_paths(base_path, patterns):
         else:
             raise Exception("%s did not match anything" % pattern)
     for path in sorted(files.keys()):
-        if path.endswith((".pyc", ".pyd", ".pyo")):
-            continue
-        h.update(
-            six.ensure_binary(
-                "{} {}\n".format(
-                    hash_path(mozpath.abspath(mozpath.join(base_path, path))),
-                    mozpath.normsep(path),
-                )
-            )
-        )
+        # If the file is dirty, read the file contents from the VCS directly.
+        # Otherwise, read the contents from the filesystem.
+        if path in dirty:
+            path_hash = hash_path_as_of_base_revision(base_path, path)
+        else:
+            path_hash = hash_path(mozpath.abspath(mozpath.join(base_path, path)))
+        h.update(six.ensure_binary("{} {}\n".format(path_hash, mozpath.normsep(path))))
+
     return h.hexdigest()