Make gclient ready for the Blink (DEPS to main project) transition

This CL makes gclient understand correctly whether a git project is
being moved from DEPS to an upper project and vice-versa.
The driving use case for this is the upcoming Blink merge, where
third_party/Webkit will be removed from DEPS (and .gitignore) and will 
become part of the main project.

At present state, gclient leaves the .git folder around when a project
is removed from DEPS, and that causes many problems. 

Furthermore this CL solves the performance problem of bisecting across
the merge point. The subproject's (Blink) .git/ folder is moved to a
backup location (in the main checkout root) and is restored when moving
backwards, avoiding a re-fetch when bisecting across the merge point. 

BUG=431469

Review URL: https://codereview.chromium.org/743083002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@293329 0039d316-1c4b-4281-b951-d872f2087c98

gclient.py

@@ -1542,23 +1542,18 @@ def RunOnDeps(self, command, args, ignore_requirements=False, progress=True):
  # Fix path separator on Windows.
  entry_fixed = entry.replace('/', os.path.sep)
  e_dir = os.path.join(self.root_dir, entry_fixed)
-
- def _IsParentOfAny(parent, path_list):
- parent_plus_slash = parent + '/'
- return any(
- path[:len(parent_plus_slash)] == parent_plus_slash
- for path in path_list)
-
  # Use entry and not entry_fixed there.
  if (entry not in entries and
  (not any(path.startswith(entry + '/') for path in entries)) and
  os.path.exists(e_dir)):
+ # The entry has been removed from DEPS.
  scm = gclient_scm.CreateSCM(
  prev_url, self.root_dir, entry_fixed, self.outbuf)
 
  # Check to see if this directory is now part of a higher-up checkout.
  # The directory might be part of a git OR svn checkout.
  scm_root = None
+ scm_class = None
  for scm_class in (gclient_scm.scm.GIT, gclient_scm.scm.SVN):
  try:
  scm_root = scm_class.GetCheckoutRoot(scm.checkout_path)
@@ -1571,9 +1566,45 @@ def _IsParentOfAny(parent, path_list):
  'determine whether it is part of a higher-level '
  'checkout, so not removing.' % entry)
  continue
+
+ # This is to handle the case of third_party/WebKit migrating from
+ # being a DEPS entry to being part of the main project.
+ # If the subproject is a Git project, we need to remove its .git
+ # folder. Otherwise git operations on that folder will have different
+ # effects depending on the current working directory.
+ if scm_class == gclient_scm.scm.GIT and (
+ os.path.abspath(scm_root) == os.path.abspath(e_dir)):
+ e_par_dir = os.path.join(e_dir, os.pardir)
+ if scm_class.IsInsideWorkTree(e_par_dir):
+ par_scm_root = scm_class.GetCheckoutRoot(e_par_dir)
+ # rel_e_dir : relative path of entry w.r.t. its parent repo.
+ rel_e_dir = os.path.relpath(e_dir, par_scm_root)
+ if scm_class.IsDirectoryVersioned(par_scm_root, rel_e_dir):
+ save_dir = scm.GetGitBackupDirPath()
+ # Remove any eventual stale backup dir for the same project.
+ if os.path.exists(save_dir):
+ gclient_utils.rmtree(save_dir)
+ os.rename(os.path.join(e_dir, '.git'), save_dir)
+ # When switching between the two states (entry/ is a subproject
+ # -> entry/ is part of the outer project), it is very likely
+ # that some files are changed in the checkout, unless we are
+ # jumping *exactly* across the commit which changed just DEPS.
+ # In such case we want to cleanup any eventual stale files
+ # (coming from the old subproject) in order to end up with a
+ # clean checkout.
+ scm_class.CleanupDir(par_scm_root, rel_e_dir)
+ assert not os.path.exists(os.path.join(e_dir, '.git'))
+ print(('\nWARNING: \'%s\' has been moved from DEPS to a higher '
+ 'level checkout. The git folder containing all the local'
+ ' branches has been saved to %s.\n'
+ 'If you don\'t care about its state you can safely '
+ 'remove that folder to free up space.') %
+ (entry, save_dir))
+ continue
+
  if scm_root in full_entries:
- logging.info('%s is part of a higher level checkout, not '
- 'removing.', scm.GetCheckoutRoot())
+ logging.info('%s is part of a higher level checkout, not removing',
+ scm.GetCheckoutRoot())
  continue
 
  file_list = []

gclient_scm.py

@@ -390,6 +390,20 @@ def update(self, options, args, file_list):
  if mirror:
  url = mirror.mirror_path
 
+ # If we are going to introduce a new project, there is a possibility that
+ # we are syncing back to a state where the project was originally a
+ # sub-project rolled by DEPS (realistic case: crossing the Blink merge point
+ # syncing backwards, when Blink was a DEPS entry and not part of src.git).
+ # In such case, we might have a backup of the former .git folder, which can
+ # be used to avoid re-fetching the entire repo again (useful for bisects).
+ backup_dir = self.GetGitBackupDirPath()
+ target_dir = os.path.join(self.checkout_path, '.git')
+ if os.path.exists(backup_dir) and not os.path.exists(target_dir):
+ gclient_utils.safe_makedirs(self.checkout_path)
+ os.rename(backup_dir, target_dir)
+ # Reset to a clean state
+ self._Run(['reset', '--hard', 'HEAD'], options)
+
  if (not os.path.exists(self.checkout_path) or
  (os.path.isdir(self.checkout_path) and
  not os.path.exists(os.path.join(self.checkout_path, '.git')))):
@@ -800,6 +814,12 @@ def FullUrlForRelativeUrl(self, url):
  base_url = self.url
  return base_url[:base_url.rfind('/')] + url
 
+ def GetGitBackupDirPath(self):
+ """Returns the path where the .git folder for the current project can be
+ staged/restored. Use case: subproject moved from DEPS <-> outer project."""
+ return os.path.join(self._root_dir,
+ 'old_' + self.relpath.replace(os.sep, '_')) + '.git'
+
  def _GetMirror(self, url, options):
  """Get a git_cache.Mirror object for the argument url."""
  if not git_cache.Mirror.GetCachePath():

scm.py

@@ -442,6 +442,16 @@ def IsInsideWorkTree(cwd):
  except (OSError, subprocess2.CalledProcessError):
  return False
 
+ @staticmethod
+ def IsDirectoryVersioned(cwd, relative_dir):
+ """Checks whether the given |relative_dir| is part of cwd's repo."""
+ return bool(GIT.Capture(['ls-tree', 'HEAD', relative_dir], cwd=cwd))
+
+ @staticmethod
+ def CleanupDir(cwd, relative_dir):
+ """Cleans up untracked file inside |relative_dir|."""
+ return bool(GIT.Capture(['clean', '-df', relative_dir], cwd=cwd))
+
  @staticmethod
  def GetGitSvnHeadRev(cwd):
  """Gets the most recently pulled git-svn revision."""

testing_support/fake_repos.py

@@ -828,6 +828,40 @@ def populateGit(self):
  })
 
 
+class FakeRepoBlinkDEPS(FakeReposBase):
+ """Simulates the Blink DEPS transition in Chrome."""
+
+ NB_GIT_REPOS = 2
+ DEPS_pre = 'deps = {"src/third_party/WebKit": "%(git_base)srepo_2",}'
+ DEPS_post = 'deps = {}'
+
+ def populateGit(self):
+ # Blink repo.
+ self._commit_git('repo_2', {
+ 'OWNERS': 'OWNERS-pre',
+ 'Source/exists_always': '_ignored_',
+ 'Source/exists_before_but_not_after': '_ignored_',
+ })
+
+ # Chrome repo.
+ self._commit_git('repo_1', {
+ 'DEPS': self.DEPS_pre % {'git_base': self.git_base},
+ 'myfile': 'myfile@1',
+ '.gitignore': '/third_party/WebKit',
+ })
+ self._commit_git('repo_1', {
+ 'DEPS': self.DEPS_post % {'git_base': self.git_base},
+ 'myfile': 'myfile@2',
+ '.gitignore': '',
+ 'third_party/WebKit/OWNERS': 'OWNERS-post',
+ 'third_party/WebKit/Source/exists_always': '_ignored_',
+ 'third_party/WebKit/Source/exists_after_but_not_before': '_ignored',
+ })
+
+ def populateSvn(self):
+ raise NotImplementedError()
+
+
 class FakeReposTestBase(trial_dir.TestCase):
  """This is vaguely inspired by twisted."""
  # Static FakeRepos instances. Lazy loaded.

tests/gclient_scm_test.py

@@ -1245,6 +1245,8 @@ def setUp(self):
  self.root_dir = '/tmp' if sys.platform != 'win32' else 't:\\tmp'
  self.relpath = 'fake'
  self.base_path = os.path.join(self.root_dir, self.relpath)
+ self.backup_base_path = os.path.join(self.root_dir,
+ 'old_%s.git' % self.relpath)
 
  def tearDown(self):
  BaseTestCase.tearDown(self)
@@ -1354,6 +1356,7 @@ def testUpdateNoDotGit(self):
 
  gclient_scm.os.path.isdir(
  os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False)
+ gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False)
  gclient_scm.os.path.exists(self.base_path).AndReturn(True)
  gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
  gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')
@@ -1384,6 +1387,7 @@ def testUpdateConflict(self):
 
  gclient_scm.os.path.isdir(
  os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False)
+ gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False)
  gclient_scm.os.path.exists(self.base_path).AndReturn(True)
  gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
  gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')

tests/gclient_smoketest.py

@@ -22,7 +22,7 @@
 
 from testing_support.fake_repos import join, write
 from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive, \
- FakeRepoSkiaDEPS
+ FakeRepoSkiaDEPS, FakeRepoBlinkDEPS
 
 import gclient_utils
 import scm as gclient_scm
@@ -1538,6 +1538,136 @@ def testSkiaDEPSChangeGit(self):
  skia_src), src_git_url)
 
 
+class BlinkDEPSTransitionSmokeTest(GClientSmokeBase):
+ """Simulate the behavior of bisect bots as they transition across the Blink
+ DEPS change."""
+
+ FAKE_REPOS_CLASS = FakeRepoBlinkDEPS
+
+ def setUp(self):
+ super(BlinkDEPSTransitionSmokeTest, self).setUp()
+ self.enabled = self.FAKE_REPOS.set_up_git()
+ self.checkout_path = os.path.join(self.root_dir, 'src')
+ self.blink = os.path.join(self.checkout_path, 'third_party', 'WebKit')
+ self.blink_git_url = self.FAKE_REPOS.git_base + 'repo_2'
+ self.pre_merge_sha = self.githash('repo_1', 1)
+ self.post_merge_sha = self.githash('repo_1', 2)
+
+ def CheckStatusPreMergePoint(self):
+ self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+ self.blink), self.blink_git_url)
+ self.assertTrue(os.path.exists(join(self.blink, '.git')))
+ self.assertTrue(os.path.exists(join(self.blink, 'OWNERS')))
+ with open(join(self.blink, 'OWNERS')) as f:
+ owners_content = f.read()
+ self.assertEqual('OWNERS-pre', owners_content, 'OWNERS not updated')
+ self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always')))
+ self.assertTrue(os.path.exists(
+ join(self.blink, 'Source', 'exists_before_but_not_after')))
+ self.assertFalse(os.path.exists(
+ join(self.blink, 'Source', 'exists_after_but_not_before')))
+
+ def CheckStatusPostMergePoint(self):
+ # Check that the contents still exists
+ self.assertTrue(os.path.exists(join(self.blink, 'OWNERS')))
+ with open(join(self.blink, 'OWNERS')) as f:
+ owners_content = f.read()
+ self.assertEqual('OWNERS-post', owners_content, 'OWNERS not updated')
+ self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always')))
+ # Check that file removed between the branch point are actually deleted.
+ self.assertTrue(os.path.exists(
+ join(self.blink, 'Source', 'exists_after_but_not_before')))
+ self.assertFalse(os.path.exists(
+ join(self.blink, 'Source', 'exists_before_but_not_after')))
+ # But not the .git folder
+ self.assertFalse(os.path.exists(join(self.blink, '.git')))
+
+ def testBlinkDEPSChangeUsingGclient(self):
+ """Checks that {src,blink} repos are consistent when syncing going back and
+ forth using gclient sync src@revision."""
+ if not self.enabled:
+ return
+
+ self.gclient(['config', '--spec',
+ 'solutions=['
+ '{"name": "src",'
+ ' "url": "' + self.git_base + 'repo_1",'
+ '}]'])
+
+ # Go back and forth two times.
+ for _ in xrange(2):
+ res = self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha])
+ self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+ self.CheckStatusPreMergePoint()
+
+ res = self.gclient(['sync', '--revision', 'src@%s' % self.post_merge_sha])
+ self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+ self.CheckStatusPostMergePoint()
+
+
+ def testBlinkDEPSChangeUsingGit(self):
+ """Like testBlinkDEPSChangeGit, but move the main project using directly
+ git and not gclient sync."""
+ if not self.enabled:
+ return
+
+ self.gclient(['config', '--spec',
+ 'solutions=['
+ '{"name": "src",'
+ ' "url": "' + self.git_base + 'repo_1",'
+ ' "managed": False,'
+ '}]'])
+
+ # Perform an initial sync to bootstrap the repo.
+ res = self.gclient(['sync'])
+ self.assertEqual(res[2], 0, 'Initial gclient sync failed.')
+
+ # Go back and forth two times.
+ for _ in xrange(2):
+ subprocess2.check_call(['git', 'checkout', '-q', self.pre_merge_sha],
+ cwd=self.checkout_path)
+ res = self.gclient(['sync'])
+ self.assertEqual(res[2], 0, 'gclient sync failed.')
+ self.CheckStatusPreMergePoint()
+
+ subprocess2.check_call(['git', 'checkout', '-q', self.post_merge_sha],
+ cwd=self.checkout_path)
+ res = self.gclient(['sync'])
+ self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+ self.CheckStatusPostMergePoint()
+
+
+ def testBlinkLocalBranchesArePreserved(self):
+ """Checks that the state of local git branches are effectively preserved
+ when going back and forth."""
+ if not self.enabled:
+ return
+
+ self.gclient(['config', '--spec',
+ 'solutions=['
+ '{"name": "src",'
+ ' "url": "' + self.git_base + 'repo_1",'
+ '}]'])
+
+ # Initialize to pre-merge point.
+ self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha])
+ self.CheckStatusPreMergePoint()
+
+ # Create a branch named "foo".
+ subprocess2.check_call(['git', 'checkout', '-qB', 'foo'],
+ cwd=self.blink)
+
+ # Cross the pre-merge point.
+ self.gclient(['sync', '--revision', 'src@%s' % self.post_merge_sha])
+ self.CheckStatusPostMergePoint()
+
+ # Go backwards and check that we still have the foo branch.
+ self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha])
+ self.CheckStatusPreMergePoint()
+ subprocess2.check_call(
+ ['git', 'show-ref', '-q', '--verify', 'refs/heads/foo'], cwd=self.blink)
+
+
 class GClientSmokeFromCheckout(GClientSmokeBase):
  # WebKit abuses this. It has a .gclient and a DEPS from a checkout.
  def setUp(self):

tests/scm_unittest.py

@@ -77,6 +77,7 @@ def testMembersChanged(self):
  'AssertVersion',
  'Capture',
  'CaptureStatus',
+ 'CleanupDir',
  'current_version',
  'FetchUpstreamTuple',
  'GenerateDiff',
@@ -92,6 +93,7 @@ def testMembersChanged(self):
  'GetSha1ForSvnRev',
  'GetSVNBranch',
  'GetUpstreamBranch',
+ 'IsDirectoryVersioned',
  'IsGitSvn',
  'IsInsideWorkTree',
  'IsValidRevision',