-
Notifications
You must be signed in to change notification settings - Fork 0
/
gclient.py
1231 lines (1094 loc) · 47 KB
/
gclient.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/python
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Meta checkout manager supporting both Subversion and GIT.
Files
.gclient : Current client configuration, written by 'config' command.
Format is a Python script defining 'solutions', a list whose
entries each are maps binding the strings "name" and "url"
to strings specifying the name and location of the client
module, as well as "custom_deps" to a map similar to the DEPS
file below.
.gclient_entries : A cache constructed by 'update' command. Format is a
Python script defining 'entries', a list of the names
of all modules in the client
<module>/DEPS : Python script defining var 'deps' as a map from each requisite
submodule name to a URL where it can be found (via one SCM)
Hooks
.gclient and DEPS files may optionally contain a list named "hooks" to
allow custom actions to be performed based on files that have changed in the
working copy as a result of a "sync"/"update" or "revert" operation. This
can be prevented by using --nohooks (hooks run by default). Hooks can also
be forced to run with the "runhooks" operation. If "sync" is run with
--force, all known hooks will run regardless of the state of the working
copy.
Each item in a "hooks" list is a dict, containing these two keys:
"pattern" The associated value is a string containing a regular
expression. When a file whose pathname matches the expression
is checked out, updated, or reverted, the hook's "action" will
run.
"action" A list describing a command to run along with its arguments, if
any. An action command will run at most one time per gclient
invocation, regardless of how many files matched the pattern.
The action is executed in the same directory as the .gclient
file. If the first item in the list is the string "python",
the current Python interpreter (sys.executable) will be used
to run the command. If the list contains string "$matching_files"
it will be removed from the list and the list will be extended
by the list of matching files.
Example:
hooks = [
{ "pattern": "\\.(gif|jpe?g|pr0n|png)$",
"action": ["python", "image_indexer.py", "--all"]},
]
"""
__version__ = "0.6"
import copy
import logging
import optparse
import os
import posixpath
import pprint
import re
import subprocess
import sys
import urlparse
import urllib
import breakpad
import gclient_scm
import gclient_utils
from third_party.repo.progress import Progress
def attr(attr, data):
"""Sets an attribute on a function."""
def hook(fn):
setattr(fn, attr, data)
return fn
return hook
## GClient implementation.
class GClientKeywords(object):
class FromImpl(object):
"""Used to implement the From() syntax."""
def __init__(self, module_name, sub_target_name=None):
"""module_name is the dep module we want to include from. It can also be
the name of a subdirectory to include from.
sub_target_name is an optional parameter if the module name in the other
DEPS file is different. E.g., you might want to map src/net to net."""
self.module_name = module_name
self.sub_target_name = sub_target_name
def __str__(self):
return 'From(%s, %s)' % (repr(self.module_name),
repr(self.sub_target_name))
class FileImpl(object):
"""Used to implement the File('') syntax which lets you sync a single file
from a SVN repo."""
def __init__(self, file_location):
self.file_location = file_location
def __str__(self):
return 'File("%s")' % self.file_location
def GetPath(self):
return os.path.split(self.file_location)[0]
def GetFilename(self):
rev_tokens = self.file_location.split('@')
return os.path.split(rev_tokens[0])[1]
def GetRevision(self):
rev_tokens = self.file_location.split('@')
if len(rev_tokens) > 1:
return rev_tokens[1]
return None
class VarImpl(object):
def __init__(self, custom_vars, local_scope):
self._custom_vars = custom_vars
self._local_scope = local_scope
def Lookup(self, var_name):
"""Implements the Var syntax."""
if var_name in self._custom_vars:
return self._custom_vars[var_name]
elif var_name in self._local_scope.get("vars", {}):
return self._local_scope["vars"][var_name]
raise gclient_utils.Error("Var is not defined: %s" % var_name)
class Dependency(GClientKeywords, gclient_utils.WorkItem):
"""Object that represents a dependency checkout."""
DEPS_FILE = 'DEPS'
def __init__(self, parent, name, url, safesync_url, custom_deps,
custom_vars, deps_file, should_process):
GClientKeywords.__init__(self)
self.parent = parent
self.name = name
self.url = url
self.parsed_url = None
# These 2 are only set in .gclient and not in DEPS files.
self.safesync_url = safesync_url
self.custom_vars = custom_vars or {}
self.custom_deps = custom_deps or {}
self.deps_hooks = []
self.dependencies = []
self.deps_file = deps_file or self.DEPS_FILE
# A cache of the files affected by the current operation, necessary for
# hooks.
self._file_list = []
# If it is not set to True, the dependency wasn't processed for its child
# dependency, i.e. its DEPS wasn't read.
self.deps_parsed = False
# This dependency should be processed, i.e. checked out
self.should_process = should_process
# This dependency has been processed, i.e. checked out
self.processed = False
# This dependency had its hook run
self.hooks_ran = False
# Required dependencies to run before running this one:
self.requirements = []
if self.parent and self.parent.name:
self.requirements.append(self.parent.name)
if isinstance(self.url, self.FromImpl):
self.requirements.append(self.url.module_name)
# Sanity checks
if not self.name and self.parent:
raise gclient_utils.Error('Dependency without name')
if not isinstance(self.url,
(basestring, self.FromImpl, self.FileImpl, None.__class__)):
raise gclient_utils.Error('dependency url must be either a string, None, '
'File() or From() instead of %s' %
self.url.__class__.__name__)
if '/' in self.deps_file or '\\' in self.deps_file:
raise gclient_utils.Error('deps_file name must not be a path, just a '
'filename. %s' % self.deps_file)
def LateOverride(self, url):
"""Resolves the parsed url from url.
Manages From() keyword accordingly. Do not touch self.parsed_url nor
self.url because it may called with other urls due to From()."""
assert self.parsed_url == None or not self.should_process, self.parsed_url
overriden_url = self.get_custom_deps(self.name, url)
if overriden_url != url:
logging.info('%s, %s was overriden to %s' % (self.name, url,
overriden_url))
return overriden_url
elif isinstance(url, self.FromImpl):
ref = [dep for dep in self.tree(True) if url.module_name == dep.name]
if not ref:
raise gclient_utils.Error('Failed to find one reference to %s. %s' % (
url.module_name, ref))
# It may happen that len(ref) > 1 but it's no big deal.
ref = ref[0]
sub_target = url.sub_target_name or self.name
# Make sure the referenced dependency DEPS file is loaded and file the
# inner referenced dependency.
ref.ParseDepsFile()
found_dep = None
for d in ref.dependencies:
if d.name == sub_target:
found_dep = d
break
if not found_dep:
raise gclient_utils.Error(
'Couldn\'t find %s in %s, referenced by %s\n%s' % (
sub_target, ref.name, self.name, str(self.root_parent())))
# Call LateOverride() again.
parsed_url = found_dep.LateOverride(found_dep.url)
logging.info('%s, %s to %s' % (self.name, url, parsed_url))
return parsed_url
elif isinstance(url, basestring):
parsed_url = urlparse.urlparse(url)
if not parsed_url[0]:
# A relative url. Fetch the real base.
path = parsed_url[2]
if not path.startswith('/'):
raise gclient_utils.Error(
'relative DEPS entry \'%s\' must begin with a slash' % url)
# Create a scm just to query the full url.
parent_url = self.parent.parsed_url
if isinstance(parent_url, self.FileImpl):
parent_url = parent_url.file_location
scm = gclient_scm.CreateSCM(parent_url, self.root_dir(), None)
parsed_url = scm.FullUrlForRelativeUrl(url)
else:
parsed_url = url
logging.info('%s, %s -> %s' % (self.name, url, parsed_url))
return parsed_url
elif isinstance(url, self.FileImpl):
parsed_url = url
logging.info('%s, %s -> %s (File)' % (self.name, url, parsed_url))
return parsed_url
elif url is None:
return None
else:
raise gclient_utils.Error('Unkown url type')
def ParseDepsFile(self):
"""Parses the DEPS file for this dependency."""
assert self.processed == True
if self.deps_parsed:
logging.debug('%s was already parsed' % self.name)
return
self.deps_parsed = True
filepath = os.path.join(self.root_dir(), self.name, self.deps_file)
if not os.path.isfile(filepath):
logging.info('%s: No DEPS file found at %s' % (self.name, filepath))
return
deps_content = gclient_utils.FileRead(filepath)
logging.debug(deps_content)
# Eval the content.
# One thing is unintuitive, vars= {} must happen before Var() use.
local_scope = {}
var = self.VarImpl(self.custom_vars, local_scope)
global_scope = {
'File': self.FileImpl,
'From': self.FromImpl,
'Var': var.Lookup,
'deps_os': {},
}
try:
exec(deps_content, global_scope, local_scope)
except SyntaxError, e:
gclient_utils.SyntaxErrorToError(filepath, e)
deps = local_scope.get('deps', {})
# load os specific dependencies if defined. these dependencies may
# override or extend the values defined by the 'deps' member.
if 'deps_os' in local_scope:
for deps_os_key in self.enforced_os():
os_deps = local_scope['deps_os'].get(deps_os_key, {})
if len(self.enforced_os()) > 1:
# Ignore any conflict when including deps for more than one
# platform, so we collect the broadest set of dependencies available.
# We may end up with the wrong revision of something for our
# platform, but this is the best we can do.
deps.update([x for x in os_deps.items() if not x[0] in deps])
else:
deps.update(os_deps)
self.deps_hooks.extend(local_scope.get('hooks', []))
# If a line is in custom_deps, but not in the solution, we want to append
# this line to the solution.
for d in self.custom_deps:
if d not in deps:
deps[d] = self.custom_deps[d]
# If use_relative_paths is set in the DEPS file, regenerate
# the dictionary using paths relative to the directory containing
# the DEPS file.
use_relative_paths = local_scope.get('use_relative_paths', False)
if use_relative_paths:
rel_deps = {}
for d, url in deps.items():
# normpath is required to allow DEPS to use .. in their
# dependency local path.
rel_deps[os.path.normpath(os.path.join(self.name, d))] = url
deps = rel_deps
# Convert the deps into real Dependency.
for name, url in deps.iteritems():
if name in [s.name for s in self.dependencies]:
raise gclient_utils.Error(
'The same name "%s" appears multiple times in the deps section' %
name)
should_process = self.recursion_limit() > 0 and self.should_process
if should_process:
tree = dict((d.name, d) for d in self.tree(False))
if name in tree:
if url == tree[name].url:
logging.info('Won\'t process duplicate dependency %s' % tree[name])
# In theory we could keep it as a shadow of the other one. In
# practice, simply ignore it.
#should_process = False
continue
else:
raise gclient_utils.Error(
'Dependency %s specified more than once:\n %s\nvs\n %s' %
(name, tree[name].hierarchy(), self.hierarchy()))
self.dependencies.append(Dependency(self, name, url, None, None, None,
None, should_process))
logging.debug('Loaded: %s' % str(self))
def run(self, options, revision_overrides, command, args, work_queue):
"""Runs 'command' before parsing the DEPS in case it's a initial checkout
or a revert."""
assert self._file_list == []
if not self.should_process:
return
# When running runhooks, there's no need to consult the SCM.
# All known hooks are expected to run unconditionally regardless of working
# copy state, so skip the SCM status check.
run_scm = command not in ('runhooks', None)
self.parsed_url = self.LateOverride(self.url)
if run_scm and self.parsed_url:
if isinstance(self.parsed_url, self.FileImpl):
# Special support for single-file checkout.
if not command in (None, 'cleanup', 'diff', 'pack', 'status'):
options.revision = self.parsed_url.GetRevision()
scm = gclient_scm.SVNWrapper(self.parsed_url.GetPath(),
self.root_dir(),
self.name)
scm.RunCommand('updatesingle', options,
args + [self.parsed_url.GetFilename()],
self._file_list)
else:
# Create a shallow copy to mutate revision.
options = copy.copy(options)
options.revision = revision_overrides.get(self.name)
scm = gclient_scm.CreateSCM(self.parsed_url, self.root_dir(), self.name)
scm.RunCommand(command, options, args, self._file_list)
self._file_list = [os.path.join(self.name, f.strip())
for f in self._file_list]
self.processed = True
if self.recursion_limit() > 0:
# Then we can parse the DEPS file.
self.ParseDepsFile()
# Adjust the implicit dependency requirement; e.g. if a DEPS file contains
# both src/foo and src/foo/bar, src/foo/bar is implicitly dependent of
# src/foo. Yes, it's O(n^2)... It's important to do that before
# enqueueing them.
for s in self.dependencies:
for s2 in self.dependencies:
if s is s2:
continue
if s.name.startswith(posixpath.join(s2.name, '')):
s.requirements.append(s2.name)
# Parse the dependencies of this dependency.
for s in self.dependencies:
work_queue.enqueue(s)
def RunHooksRecursively(self, options):
"""Evaluates all hooks, running actions as needed. run()
must have been called before to load the DEPS."""
assert self.hooks_ran == False
if not self.should_process or self.recursion_limit() <= 0:
# Don't run the hook when it is above recursion_limit.
return
# If "--force" was specified, run all hooks regardless of what files have
# changed.
if self.deps_hooks:
# TODO(maruel): If the user is using git or git-svn, then we don't know
# what files have changed so we always run all hooks. It'd be nice to fix
# that.
if (options.force or
isinstance(self.parsed_url, self.FileImpl) or
gclient_scm.GetScmName(self.parsed_url) in ('git', None) or
os.path.isdir(os.path.join(self.root_dir(), self.name, '.git'))):
for hook_dict in self.deps_hooks:
self._RunHookAction(hook_dict, [])
else:
# TODO(phajdan.jr): We should know exactly when the paths are absolute.
# Convert all absolute paths to relative.
file_list = self.file_list()
for i in range(len(file_list)):
# It depends on the command being executed (like runhooks vs sync).
if not os.path.isabs(file_list[i]):
continue
prefix = os.path.commonprefix([self.root_dir().lower(),
file_list[i].lower()])
file_list[i] = file_list[i][len(prefix):]
# Strip any leading path separators.
while (file_list[i].startswith('\\') or
file_list[i].startswith('/')):
file_list[i] = file_list[i][1:]
# Run hooks on the basis of whether the files from the gclient operation
# match each hook's pattern.
for hook_dict in self.deps_hooks:
pattern = re.compile(hook_dict['pattern'])
matching_file_list = [f for f in file_list if pattern.search(f)]
if matching_file_list:
self._RunHookAction(hook_dict, matching_file_list)
for s in self.dependencies:
s.RunHooksRecursively(options)
def _RunHookAction(self, hook_dict, matching_file_list):
"""Runs the action from a single hook."""
# A single DEPS file can specify multiple hooks so this function can be
# called multiple times on a single Dependency.
#assert self.hooks_ran == False
self.hooks_ran = True
logging.debug(hook_dict)
logging.debug(matching_file_list)
command = hook_dict['action'][:]
if command[0] == 'python':
# If the hook specified "python" as the first item, the action is a
# Python script. Run it by starting a new copy of the same
# interpreter.
command[0] = sys.executable
if '$matching_files' in command:
splice_index = command.index('$matching_files')
command[splice_index:splice_index + 1] = matching_file_list
# Use a discrete exit status code of 2 to indicate that a hook action
# failed. Users of this script may wish to treat hook action failures
# differently from VC failures.
return gclient_utils.SubprocessCall(command, self.root_dir(), fail_status=2)
def root_dir(self):
return self.parent.root_dir()
def enforced_os(self):
return self.parent.enforced_os()
def recursion_limit(self):
return self.parent.recursion_limit() - 1
def tree(self, include_all):
return self.parent.tree(include_all)
def subtree(self, include_all):
"""Breadth first"""
result = []
for d in self.dependencies:
if d.should_process or include_all:
result.append(d)
for d in self.dependencies:
result.extend(d.subtree(include_all))
return result
def get_custom_deps(self, name, url):
"""Returns a custom deps if applicable."""
if self.parent:
url = self.parent.get_custom_deps(name, url)
# None is a valid return value to disable a dependency.
return self.custom_deps.get(name, url)
def file_list(self):
result = self._file_list[:]
for d in self.dependencies:
result.extend(d.file_list())
return result
def __str__(self):
out = []
for i in ('name', 'url', 'parsed_url', 'safesync_url', 'custom_deps',
'custom_vars', 'deps_hooks', '_file_list', 'should_process',
'processed', 'hooks_ran', 'deps_parsed', 'requirements'):
# 'deps_file'
if self.__dict__[i]:
out.append('%s: %s' % (i, self.__dict__[i]))
for d in self.dependencies:
out.extend([' ' + x for x in str(d).splitlines()])
out.append('')
return '\n'.join(out)
def __repr__(self):
return '%s: %s' % (self.name, self.url)
def hierarchy(self):
"""Returns a human-readable hierarchical reference to a Dependency."""
out = '%s(%s)' % (self.name, self.url)
i = self.parent
while i and i.name:
out = '%s(%s) -> %s' % (i.name, i.url, out)
i = i.parent
return out
def root_parent(self):
"""Returns the root object, normally a GClient object."""
d = self
while d.parent:
d = d.parent
return d
class GClient(Dependency):
"""Object that represent a gclient checkout. A tree of Dependency(), one per
solution or DEPS entry."""
DEPS_OS_CHOICES = {
"win32": "win",
"win": "win",
"cygwin": "win",
"darwin": "mac",
"mac": "mac",
"unix": "unix",
"linux": "unix",
"linux2": "unix",
}
DEFAULT_CLIENT_FILE_TEXT = ("""\
solutions = [
{ "name" : "%(solution_name)s",
"url" : "%(solution_url)s",
"custom_deps" : {
},
"safesync_url": "%(safesync_url)s",
},
]
""")
DEFAULT_SNAPSHOT_SOLUTION_TEXT = ("""\
{ "name" : "%(solution_name)s",
"url" : "%(solution_url)s",
"custom_deps" : {
%(solution_deps)s },
"safesync_url": "%(safesync_url)s",
},
""")
DEFAULT_SNAPSHOT_FILE_TEXT = ("""\
# Snapshot generated with gclient revinfo --snapshot
solutions = [
%(solution_list)s]
""")
def __init__(self, root_dir, options):
# Do not change previous behavior. Only solution level and immediate DEPS
# are processed.
self._recursion_limit = 2
Dependency.__init__(self, None, None, None, None, None, None, None, True)
self._options = options
if options.deps_os:
enforced_os = options.deps_os.split(',')
else:
enforced_os = [self.DEPS_OS_CHOICES.get(sys.platform, 'unix')]
if 'all' in enforced_os:
enforced_os = self.DEPS_OS_CHOICES.itervalues()
self._enforced_os = list(set(enforced_os))
self._root_dir = root_dir
self.config_content = None
def SetConfig(self, content):
assert self.dependencies == []
config_dict = {}
self.config_content = content
try:
exec(content, config_dict)
except SyntaxError, e:
gclient_utils.SyntaxErrorToError('.gclient', e)
for s in config_dict.get('solutions', []):
try:
tree = dict((d.name, d) for d in self.tree(False))
if s['name'] in tree:
raise gclient_utils.Error(
'Dependency %s specified more than once in .gclient' % s['name'])
self.dependencies.append(Dependency(
self, s['name'], s['url'],
s.get('safesync_url', None),
s.get('custom_deps', {}),
s.get('custom_vars', {}),
None,
True))
except KeyError:
raise gclient_utils.Error('Invalid .gclient file. Solution is '
'incomplete: %s' % s)
# .gclient can have hooks.
self.deps_hooks = config_dict.get('hooks', [])
self.direct_reference = True
self.deps_parsed = True
def SaveConfig(self):
gclient_utils.FileWrite(os.path.join(self.root_dir(),
self._options.config_filename),
self.config_content)
@staticmethod
def LoadCurrentConfig(options):
"""Searches for and loads a .gclient file relative to the current working
dir. Returns a GClient object."""
path = gclient_utils.FindGclientRoot(os.getcwd(), options.config_filename)
if not path:
return None
client = GClient(path, options)
client.SetConfig(gclient_utils.FileRead(
os.path.join(path, options.config_filename)))
return client
def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % {
'solution_name': solution_name,
'solution_url': solution_url,
'safesync_url' : safesync_url,
})
def _SaveEntries(self):
"""Creates a .gclient_entries file to record the list of unique checkouts.
The .gclient_entries file lives in the same directory as .gclient.
"""
# Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It
# makes testing a bit too fun.
result = 'entries = {\n'
for entry in self.tree(False):
# Skip over File() dependencies as we can't version them.
if not isinstance(entry.parsed_url, self.FileImpl):
result += ' %s: %s,\n' % (pprint.pformat(entry.name),
pprint.pformat(entry.parsed_url))
result += '}\n'
file_path = os.path.join(self.root_dir(), self._options.entries_filename)
logging.info(result)
gclient_utils.FileWrite(file_path, result)
def _ReadEntries(self):
"""Read the .gclient_entries file for the given client.
Returns:
A sequence of solution names, which will be empty if there is the
entries file hasn't been created yet.
"""
scope = {}
filename = os.path.join(self.root_dir(), self._options.entries_filename)
if not os.path.exists(filename):
return {}
try:
exec(gclient_utils.FileRead(filename), scope)
except SyntaxError, e:
gclient_utils.SyntaxErrorToError(filename, e)
return scope['entries']
def _EnforceRevisions(self):
"""Checks for revision overrides."""
revision_overrides = {}
if self._options.head:
return revision_overrides
for s in self.dependencies:
if not s.safesync_url:
continue
handle = urllib.urlopen(s.safesync_url)
rev = handle.read().strip()
handle.close()
if len(rev):
self._options.revisions.append('%s@%s' % (s.name, rev))
if not self._options.revisions:
return revision_overrides
# --revision will take over safesync_url.
solutions_names = [s.name for s in self.dependencies]
index = 0
for revision in self._options.revisions:
if not '@' in revision:
# Support for --revision 123
revision = '%s@%s' % (solutions_names[index], revision)
sol, rev = revision.split('@', 1)
if not sol in solutions_names:
#raise gclient_utils.Error('%s is not a valid solution.' % sol)
print >> sys.stderr, ('Please fix your script, having invalid '
'--revision flags will soon considered an error.')
else:
revision_overrides[sol] = rev
index += 1
return revision_overrides
def RunOnDeps(self, command, args):
"""Runs a command on each dependency in a client and its dependencies.
Args:
command: The command to use (e.g., 'status' or 'diff')
args: list of str - extra arguments to add to the command line.
"""
if not self.dependencies:
raise gclient_utils.Error('No solution specified')
revision_overrides = self._EnforceRevisions()
pm = None
if command == 'update' and not self._options.verbose:
pm = Progress('Syncing projects', 1)
work_queue = gclient_utils.ExecutionQueue(self._options.jobs, pm)
for s in self.dependencies:
work_queue.enqueue(s)
work_queue.flush(self._options, revision_overrides, command, args,
work_queue)
# Once all the dependencies have been processed, it's now safe to run the
# hooks.
if not self._options.nohooks:
self.RunHooksRecursively(self._options)
if command == 'update':
# Notify the user if there is an orphaned entry in their working copy.
# Only delete the directory if there are no changes in it, and
# delete_unversioned_trees is set to true.
entries = [i.name for i in self.tree(False)]
for entry, prev_url in self._ReadEntries().iteritems():
# Fix path separator on Windows.
entry_fixed = entry.replace('/', os.path.sep)
e_dir = os.path.join(self.root_dir(), entry_fixed)
# Use entry and not entry_fixed there.
if entry not in entries and os.path.exists(e_dir):
file_list = []
scm = gclient_scm.CreateSCM(prev_url, self.root_dir(), entry_fixed)
scm.status(self._options, [], file_list)
modified_files = file_list != []
if not self._options.delete_unversioned_trees or modified_files:
# There are modified files in this entry. Keep warning until
# removed.
print(('\nWARNING: \'%s\' is no longer part of this client. '
'It is recommended that you manually remove it.\n') %
entry_fixed)
else:
# Delete the entry
print('\n________ deleting \'%s\' in \'%s\'' % (
entry_fixed, self.root_dir()))
gclient_utils.RemoveDirectory(e_dir)
# record the current list of entries for next time
self._SaveEntries()
return 0
def PrintRevInfo(self):
if not self.dependencies:
raise gclient_utils.Error('No solution specified')
# Load all the settings.
work_queue = gclient_utils.ExecutionQueue(self._options.jobs, None)
for s in self.dependencies:
work_queue.enqueue(s)
work_queue.flush(self._options, {}, None, [], work_queue)
def GetURLAndRev(dep):
"""Returns the revision-qualified SCM url for a Dependency."""
if dep.parsed_url is None:
return None
if isinstance(dep.parsed_url, self.FileImpl):
original_url = dep.parsed_url.file_location
else:
original_url = dep.parsed_url
url, _ = gclient_utils.SplitUrlRevision(original_url)
scm = gclient_scm.CreateSCM(original_url, self.root_dir(), dep.name)
if not os.path.isdir(scm.checkout_path):
return None
return '%s@%s' % (url, scm.revinfo(self._options, [], None))
if self._options.snapshot:
new_gclient = ''
# First level at .gclient
for d in self.dependencies:
entries = {}
def GrabDeps(dep):
"""Recursively grab dependencies."""
for d in dep.dependencies:
entries[d.name] = GetURLAndRev(d)
GrabDeps(d)
GrabDeps(d)
custom_deps = []
for k in sorted(entries.keys()):
if entries[k]:
# Quotes aren't escaped...
custom_deps.append(' \"%s\": \'%s\',\n' % (k, entries[k]))
else:
custom_deps.append(' \"%s\": None,\n' % k)
new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % {
'solution_name': d.name,
'solution_url': d.url,
'safesync_url' : d.safesync_url or '',
'solution_deps': ''.join(custom_deps),
}
# Print the snapshot configuration file
print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient})
else:
entries = {}
for d in self.tree(False):
if self._options.actual:
entries[d.name] = GetURLAndRev(d)
else:
entries[d.name] = d.parsed_url
keys = sorted(entries.keys())
for x in keys:
print('%s: %s' % (x, entries[x]))
logging.info(str(self))
def ParseDepsFile(self):
"""No DEPS to parse for a .gclient file."""
raise gclient_utils.Error('Internal error')
def root_dir(self):
"""Root directory of gclient checkout."""
return self._root_dir
def enforced_os(self):
"""What deps_os entries that are to be parsed."""
return self._enforced_os
def recursion_limit(self):
"""How recursive can each dependencies in DEPS file can load DEPS file."""
return self._recursion_limit
def tree(self, include_all):
"""Returns a flat list of all the dependencies."""
return self.subtree(include_all)
#### gclient commands.
def CMDcleanup(parser, args):
"""Cleans up all working copies.
Mostly svn-specific. Simply runs 'svn cleanup' for each module.
"""
parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
help='override deps for the specified (comma-separated) '
'platform(s); \'all\' will process all deps_os '
'references')
(options, args) = parser.parse_args(args)
client = GClient.LoadCurrentConfig(options)
if not client:
raise gclient_utils.Error('client not configured; see \'gclient config\'')
if options.verbose:
# Print out the .gclient file. This is longer than if we just printed the
# client dict, but more legible, and it might contain helpful comments.
print(client.config_content)
return client.RunOnDeps('cleanup', args)
@attr('usage', '[command] [args ...]')
def CMDrecurse(parser, args):
"""Operates on all the entries.
Runs a shell command on all entries.
"""
# Stop parsing at the first non-arg so that these go through to the command
parser.disable_interspersed_args()
parser.add_option('-s', '--scm', action='append', default=[],
help='choose scm types to operate upon')
options, args = parser.parse_args(args)
root, entries = gclient_utils.GetGClientRootAndEntries()
scm_set = set()
for scm in options.scm:
scm_set.update(scm.split(','))
# Pass in the SCM type as an env variable
env = os.environ.copy()
for path, url in entries.iteritems():
scm = gclient_scm.GetScmName(url)
if scm_set and scm not in scm_set:
continue
dir = os.path.normpath(os.path.join(root, path))
env['GCLIENT_SCM'] = scm
env['GCLIENT_URL'] = url
subprocess.Popen(args, cwd=dir, env=env).communicate()
@attr('usage', '[url] [safesync url]')
def CMDconfig(parser, args):
"""Create a .gclient file in the current directory.
This specifies the configuration for further commands. After update/sync,
top-level DEPS files in each module are read to determine dependent
modules to operate on as well. If optional [url] parameter is
provided, then configuration is read from a specified Subversion server
URL.
"""
parser.add_option('--spec',
help='create a gclient file containing the provided '
'string. Due to Cygwin/Python brokenness, it '
'probably can\'t contain any newlines.')
parser.add_option('--name',
help='overrides the default name for the solution')
(options, args) = parser.parse_args(args)
if ((options.spec and args) or len(args) > 2 or
(not options.spec and not args)):
parser.error('Inconsistent arguments. Use either --spec or one or 2 args')
if os.path.exists(options.config_filename):
raise gclient_utils.Error('%s file already exists in the current directory'
% options.config_filename)
client = GClient('.', options)
if options.spec:
client.SetConfig(options.spec)
else:
base_url = args[0].rstrip('/')
if not options.name:
name = base_url.split('/')[-1]
else:
# specify an alternate relpath for the given URL.
name = options.name
safesync_url = ''
if len(args) > 1:
safesync_url = args[1]
client.SetDefaultConfig(name, base_url, safesync_url)
client.SaveConfig()
return 0
def CMDexport(parser, args):
"""Wrapper for svn export for all managed directories."""
parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
help='override deps for the specified (comma-separated) '
'platform(s); \'all\' will process all deps_os '
'references')
(options, args) = parser.parse_args(args)
if len(args) != 1:
raise gclient_utils.Error('Need directory name')
client = GClient.LoadCurrentConfig(options)
if not client:
raise gclient_utils.Error('client not configured; see \'gclient config\'')
if options.verbose:
# Print out the .gclient file. This is longer than if we just printed the
# client dict, but more legible, and it might contain helpful comments.
print(client.config_content)
return client.RunOnDeps('export', args)
@attr('epilog', """Example:
gclient pack > patch.txt
generate simple patch for configured client and dependences
""")
def CMDpack(parser, args):
"""Generate a patch which can be applied at the root of the tree.
Internally, runs 'svn diff'/'git diff' on each checked out module and
dependencies, and performs minimal postprocessing of the output. The
resulting patch is printed to stdout and can be applied to a freshly
checked out tree via 'patch -p0 < patchfile'.
"""
parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
help='override deps for the specified (comma-separated) '
'platform(s); \'all\' will process all deps_os '
'references')
(options, args) = parser.parse_args(args)
client = GClient.LoadCurrentConfig(options)
if not client:
raise gclient_utils.Error('client not configured; see \'gclient config\'')
if options.verbose:
# Print out the .gclient file. This is longer than if we just printed the
# client dict, but more legible, and it might contain helpful comments.
print(client.config_content)
return client.RunOnDeps('pack', args)
def CMDstatus(parser, args):
"""Show modification status for every dependencies."""
parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
help='override deps for the specified (comma-separated) '
'platform(s); \'all\' will process all deps_os '
'references')
(options, args) = parser.parse_args(args)
client = GClient.LoadCurrentConfig(options)
if not client:
raise gclient_utils.Error('client not configured; see \'gclient config\'')
if options.verbose:
# Print out the .gclient file. This is longer than if we just printed the
# client dict, but more legible, and it might contain helpful comments.
print(client.config_content)
return client.RunOnDeps('status', args)
@attr('epilog', """Examples:
gclient sync
update files from SCM according to current configuration,
*for modules which have changed since last update or sync*