Bug 1594867 - Add moz.build/backend bits to specify files that should…

… be built as a sandboxed wasm library r=firefox-build-system-reviewers,mshal Add backend stuff to build sandboxed wasm libraries. (Don't actually update any moz.build files to consume this yet.) Differential Revision: https://phabricator.services.mozilla.com/D54152
iaxat · Nov 27, 2019 · 8ce7577 · 8ce7577
1 parent 47b5ffd
commit 8ce7577
Show file tree

Hide file tree

Showing 19 changed files with 333 additions and 36 deletions.
diff --git a/config/config.mk b/config/config.mk
@@ -195,6 +195,10 @@ HOST_CXXFLAGS = $(COMPUTED_HOST_CXXFLAGS) $(_HOST_DEPEND_CFLAGS)
 HOST_C_LDFLAGS = $(COMPUTED_HOST_C_LDFLAGS)
 HOST_CXX_LDFLAGS = $(COMPUTED_HOST_CXX_LDFLAGS)
 
+WASM_CFLAGS = $(COMPUTED_WASM_CFLAGS) $(PGO_CFLAGS) $(_DEPEND_CFLAGS) $(MK_COMPILE_DEFINES)
+WASM_CXXFLAGS = $(COMPUTED_WASM_CXXFLAGS) $(PGO_CFLAGS) $(_DEPEND_CFLAGS) $(MK_COMPILE_DEFINES)
+WASM_LDFLAGS = $(COMPUTED_WASM_LDFLAGS)
+
 ifdef MOZ_LTO
 ifeq (Darwin,$(OS_TARGET))
 # When linking on macOS, debug info is not linked along with the final binary,
@@ -230,6 +234,9 @@ color_flags_vars := \
   COMPILE_CMFLAGS \
   COMPILE_CMMFLAGS \
   LDFLAGS \
+  WASM_LDFLAGS \
+  WASM_CFLAGS \
+  WASM_CXXFLAGS \
   $(NULL)
 
 ifdef MACH_STDOUT_ISATTY

diff --git a/config/rules.mk b/config/rules.mk
@@ -178,20 +178,22 @@ endif
 
 ifdef COMPILE_ENVIRONMENT
 ifndef TARGETS
-TARGETS			= $(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(HOST_PROGRAM) $(HOST_SIMPLE_PROGRAMS) $(HOST_SHARED_LIBRARY)
+TARGETS			= $(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(HOST_PROGRAM) $(HOST_SIMPLE_PROGRAMS) $(HOST_SHARED_LIBRARY) $(WASM_LIBRARY)
 endif
 
 COBJS = $(notdir $(CSRCS:.c=.$(OBJ_SUFFIX)))
+CWASMOBJS = $(notdir $(WASM_CSRCS:.c=.$(WASM_OBJ_SUFFIX)))
 SOBJS = $(notdir $(SSRCS:.S=.$(OBJ_SUFFIX)))
 # CPPSRCS can have different extensions (eg: .cpp, .cc)
 CPPOBJS = $(notdir $(addsuffix .$(OBJ_SUFFIX),$(basename $(CPPSRCS))))
+CPPWASMOBJS = $(notdir $(addsuffix .$(WASM_OBJ_SUFFIX),$(basename $(WASM_CPPSRCS))))
 CMOBJS = $(notdir $(CMSRCS:.m=.$(OBJ_SUFFIX)))
 CMMOBJS = $(notdir $(CMMSRCS:.mm=.$(OBJ_SUFFIX)))
 # ASFILES can have different extensions (.s, .asm)
 ASOBJS = $(notdir $(addsuffix .$(OBJ_SUFFIX),$(basename $(ASFILES))))
 RS_STATICLIB_CRATE_OBJ = $(addprefix lib,$(notdir $(RS_STATICLIB_CRATE_SRC:.rs=.$(LIB_SUFFIX))))
 ifndef OBJS
-_OBJS = $(COBJS) $(SOBJS) $(CPPOBJS) $(CMOBJS) $(CMMOBJS) $(ASOBJS)
+_OBJS = $(COBJS) $(SOBJS) $(CPPOBJS) $(CMOBJS) $(CMMOBJS) $(ASOBJS) $(CWASMOBJS) $(CPPWASMOBJS)
 OBJS = $(strip $(_OBJS))
 endif
 
@@ -214,8 +216,11 @@ SIMPLE_PROGRAMS :=
 HOST_SHARED_LIBRARY :=
 HOST_PROGRAM :=
 HOST_SIMPLE_PROGRAMS :=
+WASM_LIBRARY :=
 endif
 
+WASM_ARCHIVE = $(addsuffix .$(WASM_OBJ_SUFFIX),$(WASM_LIBRARY))
+
 ALL_TRASH = \
 	$(GARBAGE) $(TARGETS) $(OBJS) $(PROGOBJS) LOGS TAGS a.out \
 	$(filter-out $(ASFILES),$(OBJS:.$(OBJ_SUFFIX)=.s)) $(OBJS:.$(OBJ_SUFFIX)=.ii) \
@@ -227,7 +232,7 @@ ALL_TRASH = \
 	$(PROGRAM:$(BIN_SUFFIX)=.exp) $(SIMPLE_PROGRAMS:$(BIN_SUFFIX)=.exp) \
 	$(PROGRAM:$(BIN_SUFFIX)=.lib) $(SIMPLE_PROGRAMS:$(BIN_SUFFIX)=.lib) \
 	$(SIMPLE_PROGRAMS:$(BIN_SUFFIX)=.$(OBJ_SUFFIX)) \
-	$(wildcard gts_tmp_*) $(LIBRARY:%.a=.%.timestamp)
+	$(WASM_ARCHIVE) $(wildcard gts_tmp_*) $(LIBRARY:%.a=.%.timestamp)
 ALL_TRASH_DIRS = \
 	$(GARBAGE_DIRS) /no-such-file
 
@@ -410,7 +415,7 @@ compile:: host target
 
 host:: $(HOST_OBJS) $(HOST_PROGRAM) $(HOST_SIMPLE_PROGRAMS) $(HOST_RUST_PROGRAMS) $(HOST_RUST_LIBRARY_FILE) $(HOST_SHARED_LIBRARY)
 
-target:: $(filter-out $(MOZBUILD_NON_DEFAULT_TARGETS),$(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(RUST_LIBRARY_FILE) $(RUST_PROGRAMS))
+target:: $(filter-out $(MOZBUILD_NON_DEFAULT_TARGETS),$(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(RUST_LIBRARY_FILE) $(RUST_PROGRAMS) $(WASM_LIBRARY))
 
 ifndef LIBRARY
 ifdef OBJS
@@ -476,7 +481,7 @@ ifneq (,$(filter target,$(MAKECMDGOALS)))
 ifdef GNU_CC
 # Force rebuilding libraries and programs in both passes because each
 # pass uses different object files.
-$(PROGRAM) $(SHARED_LIBRARY) $(LIBRARY): FORCE
+$(PROGRAM) $(SHARED_LIBRARY) $(LIBRARY) $(WASM_LIBRARY): FORCE
 endif
 endif
 endif
@@ -630,6 +635,15 @@ $(LIBRARY): $(OBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(RM) $(REAL_LIBRARY)
 	$(AR) $(AR_FLAGS) $($@_$(OBJS_VAR_SUFFIX))
 
+$(WASM_ARCHIVE): $(CWASMOBJS) $(CPPWASMOBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
+	$(REPORT_BUILD_VERBOSE)
+	$(RM) $(WASM_LIBRARY).$(WASM_OBJ_SUFFIX)
+	$(WASM_CXX) $(OUTOPTION)$@ -Wl,--export-all $(WASM_LDFLAGS) $(CWASMOBJS) $(CPPWASMOBJS)
+$(WASM_LIBRARY): $(WASM_LIBRARY).$(WASM_OBJ_SUFFIX)
+	$(REPORT_BUILD)
+	$(RM) $(WASM_LIBRARY)
+	$(LUCETC) --bindings $(topsrcdir)/third_party/rust/lucet-wasi/bindings.json $(WASM_LIBRARY).$(WASM_OBJ_SUFFIX) --opt-level 2 -o $(WASM_LIBRARY)
+
 ifeq ($(OS_ARCH),WINNT)
 # Import libraries are created by the rules creating shared libraries.
 # The rules to copy them to $(DIST)/lib depend on $(IMPORT_LIBRARY),
@@ -699,6 +713,7 @@ $(basename $3$(notdir $1)).$2: $1 $$(call mkdir_deps,$$(MDDEPDIR))
 endef
 $(foreach f,$(CSRCS) $(SSRCS) $(CPPSRCS) $(CMSRCS) $(CMMSRCS) $(ASFILES),$(eval $(call src_objdep,$(f),$(OBJ_SUFFIX))))
 $(foreach f,$(HOST_CSRCS) $(HOST_CPPSRCS) $(HOST_CMSRCS) $(HOST_CMMSRCS),$(eval $(call src_objdep,$(f),$(_OBJ_SUFFIX),host_)))
+$(foreach f,$(WASM_CSRCS) $(WASM_CPPSRCS),$(eval $(call src_objdep,$(f),wasm)))
 
 # The Rust compiler only outputs library objects, and so we need different
 # mangling to generate dependency rules for it.
@@ -730,6 +745,10 @@ $(COBJS):
 	$(REPORT_BUILD_VERBOSE)
 	$(CC) $(OUTOPTION)$@ -c $(COMPILE_CFLAGS) $($(notdir $<)_FLAGS) $<
 
+$(CWASMOBJS):
+	$(REPORT_BUILD_VERBOSE)
+	$(WASM_CC) $(OUTOPTION)$@ -c $(WASM_CFLAGS) $($(notdir $<)_FLAGS) $<
+
 ifdef ASFILES
 # The AS_DASH_C_FLAG is needed cause not all assemblers (Solaris) accept
 # a '-c' flag.
@@ -786,6 +805,11 @@ $(CPPOBJS):
 	$(call BUILDSTATUS,OBJECT_FILE $@)
 	$(CCC) $(OUTOPTION)$@ -c $(COMPILE_CXXFLAGS) $($(notdir $<)_FLAGS) $<
 
+$(CPPWASMOBJS):
+	$(REPORT_BUILD_VERBOSE)
+	$(call BUILDSTATUS,OBJECT_FILE $@)
+	$(WASM_CXX) $(OUTOPTION)$@ -c $(WASM_CXXFLAGS) $($(notdir $<)_FLAGS) $<
+
 $(CMMOBJS):
 	$(REPORT_BUILD_VERBOSE)
 	$(CCC) -o $@ -c $(COMPILE_CXXFLAGS) $(COMPILE_CMMFLAGS) $($(notdir $<)_FLAGS) $<
@@ -1276,6 +1300,7 @@ FREEZE_VARIABLES = \
   EXPORTS \
   DIRS \
   LIBRARY \
+  WASM_LIBRARY \
   MODULE \
   $(NULL)
 

diff --git a/moz.configure b/moz.configure
@@ -247,6 +247,7 @@ add_old_configure_assignment('OBJ_SUFFIX', library_name_info.obj.suffix)
 set_config('IMPORT_LIB_SUFFIX', library_name_info.import_lib.suffix)
 set_define('MOZ_DLL_PREFIX', depends(library_name_info.dll.prefix)(lambda s: '"%s"' % s))
 set_define('MOZ_DLL_SUFFIX', depends(library_name_info.dll.suffix)(lambda s: '"%s"' % s))
+set_config('WASM_OBJ_SUFFIX', 'wasm')
 
 include(include_project_configure)
 

diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py
@@ -36,6 +36,7 @@
     IPDLCollection,
     LocalizedPreprocessedFiles,
     LocalizedFiles,
+    SandboxedWasmLibrary,
     SharedLibrary,
     StaticLibrary,
     UnifiedSources,
@@ -248,7 +249,8 @@ def add_objs(lib):
                     no_pgo_objs.append(o)
 
         def expand(lib, recurse_objs, system_libs):
-            if isinstance(lib, (HostLibrary, StaticLibrary)):
+            if isinstance(lib, (HostLibrary, StaticLibrary,
+                                SandboxedWasmLibrary)):
                 if lib.no_expand_lib:
                     static_libs.append(lib)
                     recurse_objs = False
@@ -271,9 +273,11 @@ def expand(lib, recurse_objs, system_libs):
 
         add_objs(input_bin)
 
-        system_libs = not isinstance(input_bin, (HostLibrary, StaticLibrary))
+        system_libs = not isinstance(input_bin, (HostLibrary, StaticLibrary,
+                                                 SandboxedWasmLibrary))
         for lib in input_bin.linked_libraries:
-            if isinstance(lib, (HostLibrary, StaticLibrary)):
+            if isinstance(lib, (HostLibrary, StaticLibrary,
+                                SandboxedWasmLibrary)):
                 expand(lib, True, system_libs)
             elif isinstance(lib, SharedLibrary):
                 if lib not in seen_libs:

diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -64,12 +64,15 @@
     HostSharedLibrary,
     RustProgram,
     RustTests,
+    SandboxedWasmLibrary,
     SharedLibrary,
     SimpleProgram,
     Sources,
     StaticLibrary,
     TestManifest,
     VariablePassthru,
+    WasmGeneratedSources,
+    WasmSources,
     XPIDLModule,
 )
 from ..util import (
@@ -513,6 +516,30 @@ def consume_object(self, obj):
                     backend_file.write('%s += %s\n' % (var, p))
             self._compile_graph[mozpath.join(
                 backend_file.relobjdir, 'host-objects')]
+        elif isinstance(obj, (WasmSources, WasmGeneratedSources)):
+            suffix_map = {
+                '.c': 'WASM_CSRCS',
+                '.cpp': 'WASM_CPPSRCS',
+            }
+            variables = [suffix_map[obj.canonical_suffix]]
+            if isinstance(obj, WasmGeneratedSources):
+                variables.append('GARBAGE')
+                base = backend_file.objdir
+                cls = ObjDirPath
+                prefix = '!'
+            else:
+                base = backend_file.srcdir
+                cls = SourcePath
+                prefix = ''
+            for f in sorted(obj.files):
+                p = self._pretty_path(
+                    cls(obj._context, prefix + mozpath.relpath(f, base)),
+                    backend_file,
+                )
+                for var in variables:
+                    backend_file.write('%s += %s\n' % (var, p))
+            self._compile_graph[mozpath.join(
+                backend_file.relobjdir, 'target-objects')]
         elif isinstance(obj, VariablePassthru):
             # Sorted so output is consistent and we don't bump mtimes.
             for k, v in sorted(obj.variables.items()):
@@ -624,6 +651,10 @@ def consume_object(self, obj):
             self._process_static_library(obj, backend_file)
             self._process_linked_libraries(obj, backend_file)
 
+        elif isinstance(obj, SandboxedWasmLibrary):
+            self._process_sandboxed_wasm_library(obj, backend_file)
+            self._process_linked_libraries(obj, backend_file)
+
         elif isinstance(obj, HostLibrary):
             self._process_linked_libraries(obj, backend_file)
 
@@ -1280,6 +1311,9 @@ def _process_static_library(self, libdef, backend_file):
         if libdef.no_expand_lib:
             backend_file.write('NO_EXPAND_LIBS := 1\n')
 
+    def _process_sandboxed_wasm_library(self, libdef, backend_file):
+        backend_file.write('WASM_LIBRARY := %s\n' % libdef.lib_name)
+
     def _process_rust_library(self, libdef, backend_file):
         backend_file.write_once('%s := %s\n' % (libdef.LIB_FILE_VAR, libdef.import_name))
         backend_file.write_once('CARGO_FILE := $(srcdir)/Cargo.toml\n')
@@ -1353,15 +1387,18 @@ def write_obj_deps(target, objs_ref, pgo_objs_ref):
         # accommodates clang-plugin, where we would otherwise pass an
         # incorrect list file format to the host compiler as well as when
         # creating an archive with AR, which doesn't understand list files.
-        if (objs == obj.objs and not isinstance(obj, (HostLibrary, StaticLibrary)) or
-            isinstance(obj, StaticLibrary) and obj.no_expand_lib):
+        if (objs == obj.objs and not isinstance(obj, (HostLibrary, StaticLibrary,
+                                                      SandboxedWasmLibrary)) or
+            isinstance(obj, (StaticLibrary, SandboxedWasmLibrary)) and
+            obj.no_expand_lib):
             backend_file.write_once('%s_OBJS := %s\n' % (obj.name,
                                                          objs_ref))
             if profile_gen_objs:
                 backend_file.write_once('%s_PGO_OBJS := %s\n' % (obj.name,
                                                                  pgo_objs_ref))
             write_obj_deps(obj_target, objs_ref, pgo_objs_ref)
-        elif not isinstance(obj, (HostLibrary, StaticLibrary)):
+        elif not isinstance(obj, (HostLibrary, StaticLibrary,
+                                  SandboxedWasmLibrary)):
             list_file_path = '%s.list' % obj.name.replace('.', '_')
             list_file_ref = self._make_list_file(obj.KIND, obj.objdir, objs,
                                                  list_file_path)

diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py
@@ -448,19 +448,20 @@ class TargetCompileFlags(BaseCompileFlags):
     """Base class that encapsulates some common logic between CompileFlags and
     WasmCompileFlags.
     """
-    def __init__(self, context, prefix='', additionally=()):
+    def __init__(self, context, prefix='', additionally=(), instead={}):
         # `prefix` is a string to be prepended to all dest_var names.
         # `additionally` is a sequence of (flat, default, dest_vars) to be added
         # to the flag_variables tuple.
+        # `instead` is a string-to-string map that maps base dest_var names to
+        # replacement names; e.g., if instead={'FOO':'BAR'}, then 'FOO' is
+        # replaced with 'BAR' wherever it would occur in a dest_var.
         main_src_dir = mozpath.dirname(context.main_path)
         self._context = context
 
         self.flag_variables = tuple(list(additionally) + [
-            (flag, default, tuple(prefix + dest_var for dest_var in dest_vars))
+            (flag, default, tuple(set(instead.get(prefix + dest_var, prefix + dest_var)
+                                      for dest_var in dest_vars)))
             for flag, default, dest_vars in (
-                    ('STL', context.config.substs.get('STL_FLAGS'),
-                     ('CXXFLAGS',)),
-                    ('DEFINES', None, ('CXXFLAGS', 'CFLAGS')),
                     ('LIBRARY_DEFINES', None, ('CXXFLAGS', 'CFLAGS')),
                     ('BASE_INCLUDES',
                      ['-I%s' % main_src_dir, '-I%s' % context.objdir],
@@ -550,6 +551,9 @@ def __setitem__(self, key, value):
 class CompileFlags(TargetCompileFlags):
     def __init__(self, context):
         TargetCompileFlags.__init__(self, context, prefix='', additionally=(
+            ('DEFINES', None, ('CXXFLAGS', 'CFLAGS')),
+            ('STL', context.config.substs.get('STL_FLAGS'),
+             ('CXXFLAGS',)),
             ('VISIBILITY', context.config.substs.get('VISIBILITY_FLAGS'),
              ('CXXFLAGS', 'CFLAGS')),
             ('MOZ_HARDENING_CFLAGS',
@@ -574,9 +578,19 @@ def __init__(self, context):
 class WasmFlags(TargetCompileFlags):
     def __init__(self, context):
         TargetCompileFlags.__init__(
-            self, context, prefix='WASM_', additionally=tuple(
-                (name, context.config.substs.get(name), (name,))
-                for name in ('WASM_CFLAGS', 'WASM_CXXFLAGS', 'WASM_LDFLAGS')))
+            self, context, prefix='WASM_', additionally=(
+                tuple(
+                    (name, context.config.substs.get(name), (name,))
+                    for name in ('WASM_CFLAGS', 'WASM_CXXFLAGS', 'WASM_LDFLAGS')) +
+                (('WASM_DEFINES', None, ('WASM_CFLAGS', 'WASM_CXXFLAGS')),
+                 ('MOZBUILD_WASM_CFLAGS', None, ('WASM_CFLAGS',)),
+                 ('MOZBUILD_WASM_CXXFLAGS', None, ('WASM_CXXFLAGS',)))),
+            # Unlike target/host compilation, we don't distinguish between
+            # LDFLAGS for C and C++.
+            instead={
+                'WASM_C_LDFLAGS': 'WASM_LDFLAGS',
+                'WASM_CXX_LDFLAGS': 'WASM_LDFLAGS',
+            })
 
 
 class FinalTargetValue(ContextDerivedValue, unicode):
@@ -1641,6 +1655,10 @@ def aggregate(files):
         with the host compiler.
         """),
 
+    'WASM_SOURCES': (ContextDerivedTypedList(Path, StrictOrderingOnAppendList), list,
+                     """Source code files to compile with the wasm compiler.
+        """),
+
     'HOST_LIBRARY_NAME': (unicode, unicode,
                           """Name of target library generated when cross compiling.
         """),
@@ -1671,6 +1689,11 @@ def aggregate(files):
         Implies FORCE_SHARED_LIB.
         """),
 
+    'SANDBOXED_WASM_LIBRARY_NAME': (
+        unicode, unicode,
+        """The name of the static sandboxed wasm library generated for a directory.
+        """),
+
     'SHARED_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode,
                                        """The output category for this context's shared library. If set this will
         correspond to the build command that will build this shared library, and
@@ -2234,6 +2257,11 @@ def aggregate(files):
            appear in the moz.build file.
         """),
 
+    'WASM_DEFINES': (InitializedDefines, dict,
+                     """Dictionary of compiler defines to declare for wasm compilation.
+        See ``DEFINES`` for specifics.
+        """),
+
     'CMFLAGS': (List, list,
                 """Flags passed to the Objective-C compiler for all of the Objective-C
            source files declared in this directory.