Bug 1474028 - Add a way to exclude libraries from the default build. …

…r=ted

MozReview-Commit-ID: MVfplx9lN2

build/gecko_templates.mozbuild

@@ -107,27 +107,27 @@ def GeckoCppUnitTests(names, **kwargs):
 
 
 @template
-def GeckoSharedLibrary(name, **kwargs):
+def GeckoSharedLibrary(name, output_category=None, **kwargs):
     '''Template for shared libraries related to Gecko.
 
     `name` identifies the library base name.
     See the documentation for `GeckoBinary` for other possible arguments.
     '''
-    SharedLibrary(name)
+    SharedLibrary(name, output_category)
 
     kwargs.setdefault('mozglue', 'library')
 
     GeckoBinary(**kwargs)
 
 
 @template
-def GeckoFramework(name, **kwargs):
+def GeckoFramework(name, output_category=None, **kwargs):
     '''Template for OSX frameworks related to Gecko.
 
     `name` identifies the library base name.
     See the documentation for `GeckoBinary` for other possible arguments.
     '''
-    Framework(name)
+    Framework(name, output_category)
 
     kwargs.setdefault('mozglue', 'library')
 

build/templates.mozbuild

@@ -58,7 +58,7 @@ def AllowCompilerWarnings():
     COMPILE_FLAGS['WARNINGS_AS_ERRORS'] = []
 
 @template
-def RustLibrary(name, features=None, target_dir=None):
+def RustLibrary(name, features=None, target_dir=None, output_category=None):
     '''Template for Rust libraries.'''
     Library(name)
 
@@ -72,21 +72,27 @@ def RustLibrary(name, features=None, target_dir=None):
     if target_dir:
         RUST_LIBRARY_TARGET_DIR = target_dir
 
+    if output_category:
+        RUST_LIBRARY_OUTPUT_CATEGORY = output_category
+
 
 @template
-def SharedLibrary(name):
+def SharedLibrary(name, output_category=None):
     '''Template for shared libraries.'''
     Library(name)
 
     FORCE_SHARED_LIB = True
 
+    if output_category:
+        SHARED_LIBRARY_OUTPUT_CATEGORY = output_category
+
     Binary()
 
 
 @template
-def Framework(name):
+def Framework(name, output_category=None):
     '''Template for OSX Frameworks.'''
-    SharedLibrary(name)
+    SharedLibrary(name, output_category)
 
     IS_FRAMEWORK = True
 

config/makefiles/target_binaries.mk

@@ -18,7 +18,9 @@ endif
 ifdef SHARED_LIBRARY
 SHARED_LIBRARY_FILES = $(SHARED_LIBRARY)
 SHARED_LIBRARY_DEST ?= $(FINAL_TARGET)
+ifndef SHARED_LIBRARY_TARGET
 SHARED_LIBRARY_TARGET = target
+endif
 INSTALL_TARGETS += SHARED_LIBRARY
 endif # SHARED_LIBRARY
 

config/recurse.mk

@@ -41,7 +41,7 @@ binaries::
 # Carefully avoid $(eval) type of rule generation, which makes pymake slower
 # than necessary.
 # Get current tier and corresponding subtiers from the data in root.mk.
-CURRENT_TIER := $(filter $(foreach tier,$(TIERS),recurse_$(tier) $(tier)-deps),$(MAKECMDGOALS))
+CURRENT_TIER := $(filter $(foreach tier,$(TIERS) $(non_default_tiers),recurse_$(tier) $(tier)-deps),$(MAKECMDGOALS))
 ifneq (,$(filter-out 0 1,$(words $(CURRENT_TIER))))
 $(error $(CURRENT_TIER) not supported on the same make command line)
 endif

config/rules.mk

@@ -446,7 +446,7 @@ compile:: host target
 
 host:: $(HOST_LIBRARY) $(HOST_PROGRAM) $(HOST_SIMPLE_PROGRAMS) $(HOST_RUST_PROGRAMS) $(HOST_RUST_LIBRARY_FILE) $(HOST_SHARED_LIBRARY)
 
-target:: $(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(RUST_LIBRARY_FILE) $(RUST_PROGRAMS)
+target:: $(filter-out $(MOZBUILD_NON_DEFAULT_TARGETS),$(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(RUST_LIBRARY_FILE) $(RUST_PROGRAMS))
 
 ifndef LIBRARY
 ifdef OBJS

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -810,12 +810,39 @@ def tools_filter(current, subdirs):
         compile_roots = [t for t, deps in self._compile_graph.iteritems()
                          if not deps or t not in all_compile_deps]
 
-        rule = root_deps_mk.create_rule(['recurse_compile'])
-        rule.add_dependencies(compile_roots)
-        for target, deps in sorted(self._compile_graph.items()):
-            if deps:
-                rule = root_deps_mk.create_rule([target])
-                rule.add_dependencies(deps)
+        def add_category_rules(category, roots, graph):
+            rule = root_deps_mk.create_rule(['recurse_%s' % category])
+            rule.add_dependencies(roots)
+            for target, deps in sorted(graph.items()):
+                if deps:
+                    rule = root_deps_mk.create_rule([target])
+                    rule.add_dependencies(deps)
+
+        non_default_roots = defaultdict(list)
+        non_default_graphs = defaultdict(lambda: OrderedDefaultDict(set))
+
+        for root in compile_roots:
+            # If this is a non-default target, separate the root from the
+            # rest of the compile graph.
+            target_name = mozpath.basename(root)
+
+            if target_name not in ('target', 'host'):
+                non_default_roots[target_name].append(root)
+                non_default_graphs[target_name][root] = self._compile_graph[root]
+                del self._compile_graph[root]
+
+        for root in chain(*non_default_roots.values()):
+            compile_roots.remove(root)
+            dirname = mozpath.dirname(root)
+            # If a directory only contains non-default compile targets, we don't
+            # attempt to dump symbols there.
+            if (dirname in self._no_skip['syms'] and
+                '%s/target' % dirname not in self._compile_graph):
+                self._no_skip['syms'].remove(dirname)
+
+        add_category_rules('compile', compile_roots, self._compile_graph)
+        for category, graph in non_default_graphs.iteritems():
+            add_category_rules(category, non_default_roots[category], graph)
 
         root_mk = Makefile()
 
@@ -831,6 +858,15 @@ def tools_filter(current, subdirs):
         root_mk.add_statement('syms_targets := %s' % ' '.join(sorted(
             set('%s/syms' % d for d in self._no_skip['syms']))))
 
+        root_mk.add_statement('non_default_tiers := %s' % ' '.join(sorted(
+            non_default_roots.keys())))
+
+        for category, graphs in non_default_graphs.iteritems():
+            category_dirs = [mozpath.dirname(target)
+                             for target in graphs.keys()]
+            root_mk.add_statement('%s_dirs := %s' % (category,
+                                                     ' '.join(category_dirs)))
+
         root_mk.add_statement('include root-deps.mk')
 
         with self._write_file(
@@ -1252,6 +1288,10 @@ def _process_computed_flags(self, computed_flags, backend_file):
             backend_file.write('COMPUTED_%s += %s\n' % (var,
                                                         ' '.join(make_quote(shell_quote(f)) for f in flags)))
 
+    def _process_non_default_target(self, libdef, target_name, backend_file):
+        backend_file.write("%s:: %s\n" % (libdef.output_category, target_name))
+        backend_file.write('MOZBUILD_NON_DEFAULT_TARGETS += %s\n' % target_name)
+
     def _process_shared_library(self, libdef, backend_file):
         backend_file.write_once('LIBRARY_NAME := %s\n' % libdef.basename)
         backend_file.write('FORCE_SHARED_LIB := 1\n')
@@ -1263,6 +1303,14 @@ def _process_shared_library(self, libdef, backend_file):
             backend_file.write('SYMBOLS_FILE := %s\n' % libdef.symbols_file)
         if not libdef.cxx_link:
             backend_file.write('LIB_IS_C_ONLY := 1\n')
+        if libdef.output_category:
+            self._process_non_default_target(libdef, libdef.lib_name,
+                                             backend_file)
+            # Override the install rule target for this library. This is hacky,
+            # but can go away as soon as we start building libraries in their
+            # final location (bug 1459764).
+            backend_file.write('SHARED_LIBRARY_TARGET := %s\n' %
+                               libdef.output_category)
 
     def _process_static_library(self, libdef, backend_file):
         backend_file.write_once('LIBRARY_NAME := %s\n' % libdef.basename)
@@ -1283,6 +1331,8 @@ def _process_rust_library(self, libdef, backend_file):
         backend_file.write('CARGO_TARGET_DIR := %s\n' % target_dir)
         if libdef.features:
             backend_file.write('%s := %s\n' % (libdef.FEATURES_VAR, ' '.join(libdef.features)))
+        if libdef.output_category:
+            self._process_non_default_target(libdef, libdef.import_name, backend_file)
 
     def _process_host_library(self, libdef, backend_file):
         backend_file.write('HOST_LIBRARY_NAME = %s\n' % libdef.basename)
@@ -1291,8 +1341,11 @@ def _process_host_shared_library(self, libdef, backend_file):
         backend_file.write('HOST_SHARED_LIBRARY = %s\n' % libdef.lib_name)
 
     def _build_target_for_obj(self, obj):
+        target_name = obj.KIND
+        if hasattr(obj, 'output_category') and obj.output_category:
+            target_name = obj.output_category
         return '%s/%s' % (mozpath.relpath(obj.objdir,
-            self.environment.topobjdir), obj.KIND)
+            self.environment.topobjdir), target_name)
 
     def _process_linked_libraries(self, obj, backend_file):
         def pretty_relpath(lib, name):

python/mozbuild/mozbuild/frontend/context.py

@@ -1552,6 +1552,18 @@ def aggregate(files):
         Implies FORCE_SHARED_LIB.
         """),
 
+    'SHARED_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode,
+        """The output category for this context's shared library. If set this will
+        correspond to the build command that will build this shared library, and
+        the library will not be built as part of the default build.
+        """),
+
+    'RUST_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode,
+        """The output category for this context's rust library. If set this will
+        correspond to the build command that will build this rust library, and
+        the library will not be built as part of the default build.
+        """),
+
     'IS_FRAMEWORK': (bool, bool,
         """Whether the library to build should be built as a framework on OSX.
 

python/mozbuild/mozbuild/frontend/data.py

@@ -673,6 +673,7 @@ class RustLibrary(StaticLibrary):
         'deps_path',
         'features',
         'target_dir',
+        'output_category',
     )
     TARGET_SUBST_VAR = 'RUST_TARGET'
     FEATURES_VAR = 'RUST_LIBRARY_FEATURES'
@@ -694,6 +695,7 @@ def __init__(self, context, basename, cargo_file, crate_type, dependencies,
         self.dependencies = dependencies
         self.features = features
         self.target_dir = target_dir
+        self.output_category = context.get('RUST_LIBRARY_OUTPUT_CATEGORY')
         # Skip setting properties below which depend on cargo
         # when we don't have a compile environment. The required
         # config keys won't be available, but the instance variables
@@ -713,6 +715,7 @@ class SharedLibrary(Library):
         'soname',
         'variant',
         'symbols_file',
+        'output_category',
     )
 
     DICT_ATTRS = {
@@ -733,6 +736,7 @@ def __init__(self, context, basename, real_name=None,
         Library.__init__(self, context, basename, real_name)
         self.variant = variant
         self.lib_name = real_name or basename
+        self.output_category = context.get('SHARED_LIBRARY_OUTPUT_CATEGORY')
         assert self.lib_name
 
         if variant == self.FRAMEWORK:

python/mozbuild/mozbuild/mach_commands.py

@@ -601,19 +601,12 @@ def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser, debug, debugger,
 
         active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
         if 'Tup' in active_backend:
-            gtest_build_path = mozpath.join(self.topobjdir, '<gtest>')
+            gtest_build_target = mozpath.join(self.topobjdir, '<gtest>')
         else:
-            # This path happens build the necessary parts of the tree in the
-            # Make backend due to the odd nature of partial tree builds.
-            gtest_build_path = mozpath.relpath(mozpath.join(self.topobjdir,
-                                                            'toolkit', 'library',
-                                                            'gtest', 'rust'),
-                                               self.topsrcdir)
-
-        os.environ[b'LINK_GTEST_DURING_COMPILE'] = b'1'
+            gtest_build_target = 'recurse_gtest'
+
         res = self._mach_context.commands.dispatch('build', self._mach_context,
-                                                   what=[gtest_build_path])
-        del os.environ[b'LINK_GTEST_DURING_COMPILE']
+                                                   what=[gtest_build_target])
         if res:
             print("Could not build xul-gtest")
             return res

toolkit/library/gtest/Makefile.in

@@ -2,27 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Enforce that the clean/distclean rules removes everything that needs
-# to be removed from this directory.
-ifneq (,$(filter clean distclean,$(MAKECMDGOALS)))
-LINK_GTEST_DURING_COMPILE = 1
-endif
-
-# Don't link the gtest xul during MOZ_PROFILE_GENERATE, it doesn't get
-# used during profiling anyway.
-ifdef MOZ_PROFILE_GENERATE
-LINK_GTEST_DURING_COMPILE =
-endif
-
-ifndef LINK_GTEST_DURING_COMPILE
-# Force to not include backend.mk unless LINK_GTEST_DURING_COMPILE is set.
-# Not including backend.mk makes traversing this directory do nothing.
-STANDALONE_MAKEFILE = 1
-
-else
-
 include $(topsrcdir)/toolkit/library/libxul.mk
 
 include $(topsrcdir)/config/config.mk
-
-endif

toolkit/library/gtest/moz.build

@@ -32,7 +32,8 @@ if CONFIG['OS_ARCH'] == 'Linux' and CONFIG['OS_TARGET'] != 'Android':
 
 # This needs to come after static:xul to avoid things like libfallible coming
 # before StaticXULComponentStart.
-Libxul('xul-gtest-real')
+Libxul('xul-gtest-real',
+       output_category=None if CONFIG['LINK_GTEST_DURING_COMPILE'] else 'gtest')
 
 DIRS += [
     'static',

toolkit/library/gtest/rust/moz.build

@@ -6,4 +6,5 @@
 
 include('../../rust/gkrust-features.mozbuild')
 
-RustLibrary('gkrust-gtest', gkrust_features, '../..')
+RustLibrary('gkrust-gtest', gkrust_features, '../..',
+            output_category=None if CONFIG['LINK_GTEST_DURING_COMPILE'] else 'gtest')

toolkit/library/moz.build

@@ -12,14 +12,14 @@ def Libxul_defines():
         LIBRARY_DEFINES['STATIC_EXPORTABLE_JS_API'] = True
 
 @template
-def Libxul(name):
+def Libxul(name, output_category=None):
     if CONFIG['MOZ_WIDGET_TOOLKIT'] in ('cocoa', 'uikit'):
         # This is going to be a framework named "XUL", not an ordinary library named
         # "libxul.dylib"
-        GeckoFramework(name, linkage=None)
+        GeckoFramework(name, output_category=output_category, linkage=None)
         SHARED_LIBRARY_NAME = 'XUL'
     else:
-        GeckoSharedLibrary(name, linkage=None)
+        GeckoSharedLibrary(name, output_category=output_category, linkage=None)
         SHARED_LIBRARY_NAME = 'xul'
 
     DELAYLOAD_DLLS += [