Add Clutz to TensorBoard build

This is so we can get JavaScript protobufs. This CL also improves the
web_aspect and makes some peculiar Closure Compiler errors go away
relating to externs.

PiperOrigin-RevId: 158061198

WORKSPACE

@@ -2,11 +2,11 @@ workspace(name = "org_tensorflow")
 
 http_archive(
     name = "io_bazel_rules_closure",
-    sha256 = "edc91f556b762fc5212d1050d00b12e40dd0b0b1c1d5d96886b59e9a30a6cae4",
-    strip_prefix = "rules_closure-3f07fb6a58870afbb36051bd5d54da4479561cc6",
+    sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846",
+    strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f",
     urls = [
-        "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz",  # 2017-05-31
-        "https://github.com/bazelbuild/rules_closure/archive/3f07fb6a58870afbb36051bd5d54da4479561cc6.tar.gz",
+        "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz",  # 2017-06-02
+        "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz",
     ],
 )
 

tensorflow/tensorboard/BUILD

@@ -33,6 +33,17 @@ filegroup(
     ],
 )
 
+filegroup(
+    name = "ts_web_library_default_typings",
+    srcs = [
+        # Ordering probably matters.
+        "@com_microsoft_typescript//:lib.es6.d.ts",
+        "@io_angular_clutz//:src/resources/closure.lib.d.ts",
+        "clutz.d.ts",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(

tensorflow/tensorboard/clutz.d.ts

@@ -0,0 +1,19 @@
+// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// tslint:disable
+declare namespace ಠ_ಠ.clutz {
+  interface IteratorIterable<T> extends Iterator<T>, Iterable<T> {}
+  interface IIterableResult<T> extends IteratorResult<T> {}
+}

tensorflow/tensorboard/defs.bzl

@@ -15,3 +15,10 @@
 def tensorboard_webcomponent_library(**kwargs):
   """Rules referencing this will be deleted from the codebase soon."""
   pass
+
+def _legacy_js_impl(target, ctx):
+  return struct()
+
+legacy_js = aspect(
+    implementation=_legacy_js_impl,
+    attr_aspects=["exports"])

tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize/Vulcanize.java

@@ -24,7 +24,6 @@
 import com.google.common.base.Optional;
 import com.google.common.base.Splitter;
 import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
@@ -38,6 +37,7 @@
 import com.google.javascript.jscomp.DiagnosticGroups;
 import com.google.javascript.jscomp.DiagnosticType;
 import com.google.javascript.jscomp.JSError;
+import com.google.javascript.jscomp.ModuleIdentifier;
 import com.google.javascript.jscomp.PropertyRenamingPolicy;
 import com.google.javascript.jscomp.Result;
 import com.google.javascript.jscomp.SourceFile;
@@ -53,8 +53,10 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Deque;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -63,6 +65,7 @@
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Attribute;
 import org.jsoup.nodes.Comment;
@@ -92,6 +95,7 @@ public final class Vulcanize {
   private static final Set<String> legalese = new HashSet<>();
   private static final List<String> licenses = new ArrayList<>();
   private static final List<Webpath> stack = new ArrayList<>();
+  private static final List<SourceFile> externs = new ArrayList<>();
   private static final List<SourceFile> sourcesFromJsLibraries = new ArrayList<>();
   private static final Map<Webpath, String> sourcesFromScriptTags = new LinkedHashMap<>();
   private static final Map<Webpath, Node> sourceTags = new LinkedHashMap<>();
@@ -111,7 +115,13 @@ public static void main(String[] args) throws IOException {
     Path output = Paths.get(args[4]);
     for (int i = 5; i < args.length; i++) {
       if (args[i].endsWith(".js")) {
-        sourcesFromJsLibraries.add(SourceFile.fromFile(args[i]));
+        String code = new String(Files.readAllBytes(Paths.get(args[i])), UTF_8);
+        SourceFile sourceFile = SourceFile.fromCode(args[i], code);
+        if (code.contains("@externs")) {
+          externs.add(sourceFile);
+        } else {
+          sourcesFromJsLibraries.add(sourceFile);
+        }
         continue;
       }
       if (!args[i].endsWith(".pbtxt")) {
@@ -339,12 +349,20 @@ private static void compile() {
     options.setRemoveUnusedPrototypePropertiesInExterns(false);
     options.setRemoveUnusedClassProperties(false);
 
-    // Closure pass.
+    // Dependency management.
     options.setClosurePass(true);
     options.setManageClosureDependencies(true);
     options.getDependencyOptions().setDependencyPruning(true);
-    options.getDependencyOptions().setDependencySorting(false);
+    options.getDependencyOptions().setDependencySorting(true);
     options.getDependencyOptions().setMoocherDropping(false);
+    options.getDependencyOptions()
+        .setEntryPoints(
+            sourceTags
+                .keySet()
+                .stream()
+                .map(Webpath::toString)
+                .map(ModuleIdentifier::forFile)
+                .collect(Collectors.toList()));
 
     // Polymer pass.
     options.setPolymerVersion(1);
@@ -362,6 +380,16 @@ private static void compile() {
         new WarningsGuard() {
           @Override
           public CheckLevel level(JSError error) {
+            if (error.sourceName == null) {
+              return null;
+            }
+            if (error.sourceName.startsWith("javascript/externs")
+                || error.sourceName.contains("com_google_javascript_closure_compiler_externs")) {
+              // TODO(jart): Figure out why these "mismatch of the removeEventListener property on
+              //             type" warnings are showing up.
+              //             https://github.com/google/closure-compiler/pull/1959
+              return CheckLevel.OFF;
+            }
             if (IGNORE_PATHS_PATTERN.matcher(error.sourceName).matches()) {
               return CheckLevel.OFF;
             }
@@ -395,42 +423,39 @@ public CheckLevel level(JSError error) {
     // Compile everything into a single script.
     Compiler compiler = new Compiler();
     compiler.disableThreads();
-    Result result = compiler.compile(ImmutableList.<SourceFile>of(), sauce, options);
+    Result result = compiler.compile(externs, sauce, options);
     if (!result.success) {
       System.exit(1);
     }
     String jsBlob = compiler.toSource();
 
     // Split apart the JS blob and put it back in the original <script> locations.
+    Deque<Map.Entry<Webpath, Node>> tags = new ArrayDeque<>();
+    tags.addAll(sourceTags.entrySet());
     Matcher matcher = WEBPATH_PATTERN.matcher(jsBlob);
-    Webpath path = null;
-    String pureJsDeps = "";
-    int start = -1;
+    verify(matcher.find(), "Nothing found in compiled JS blob!");
+    Webpath path = Webpath.get(matcher.group(1));
+    int start = 0;
     while (matcher.find()) {
-      if (!sourceTags.containsKey(Webpath.get(matcher.group(1)))) {
-        continue;  // Skip over js_library dependencies, which must group at beginning of args.
-      }
-      if (path != null) {
-        swapScript(path, pureJsDeps + jsBlob.substring(start, matcher.start()));
-        pureJsDeps = "";
-      } else {
-        pureJsDeps = jsBlob.substring(0, matcher.start());
+      if (sourceTags.containsKey(path)) {
+        swapScript(tags, path, jsBlob.substring(start, matcher.start()));
+        start = matcher.start();
       }
       path = Webpath.get(matcher.group(1));
-      start = matcher.start();
-    }
-    swapScript(path, pureJsDeps + jsBlob.substring(start));
-    if (!sourceTags.isEmpty()) {
-      throw new RuntimeException("Couldn't pull out: " + ImmutableSet.copyOf(sourceTags.keySet()));
     }
+    swapScript(tags, path, jsBlob.substring(start));
+    verify(tags.isEmpty(), "<script> wasn't compiled: %s", tags);
   }
 
-  private static void swapScript(Webpath path, String script) {
-    Node tag = sourceTags.get(path);
+  private static void swapScript(
+      Deque<Map.Entry<Webpath, Node>> tags, Webpath path, String script) {
+    verify(!tags.isEmpty(), "jscomp compiled %s after last <script>?!", path);
+    Webpath want = tags.getFirst().getKey();
+    verify(path.equals(want), "<script> tag for %s should come before %s", path, want);
+    Node tag = tags.removeFirst().getValue();
     tag.replaceWith(
         new Element(Tag.valueOf("script"), tag.baseUri())
             .appendChild(new DataNode(script, tag.baseUri())));
-    sourceTags.remove(path);
   }
 
   private static void handleLicense(String text) {

tensorflow/tensorboard/vulcanize.bzl

@@ -12,26 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@io_bazel_rules_closure//closure/private:defs.bzl", "unfurl", "long_path")
+load("//tensorflow/tensorboard:defs.bzl", "legacy_js")
+load("@io_bazel_rules_closure//closure/private:defs.bzl", "collect_js", "unfurl", "long_path")
 load("//tensorflow/tensorboard:web.bzl", "web_aspect")
 
 def _tensorboard_html_binary(ctx):
   deps = unfurl(ctx.attr.deps, provider="webfiles")
-  manifests = depset(order="topological")
-  files = depset()
-  jslibs = depset(ctx.files._jslibs)
-  webpaths = depset()
+  manifests = set(order="topological")
+  files = set()
+  webpaths = set()
   for dep in deps:
     manifests += dep.webfiles.manifests
     webpaths += dep.webfiles.webpaths
     files += dep.data_runfiles.files
-    if hasattr(dep.webfiles, "jslibs"):
-      jslibs += dep.webfiles.jslibs
-    if hasattr(dep, "closure_js_library"):
-      jslibs += getattr(dep.closure_js_library, "srcs", [])
   webpaths += [ctx.attr.output_path]
+  closure_js_library=collect_js(
+      ctx, unfurl(ctx.attr.deps, provider="closure_js_library"))
 
   # vulcanize
+  jslibs = depset(ctx.files._jslibs) + closure_js_library.srcs
   ctx.action(
       inputs=list(manifests | files | jslibs),
       outputs=[ctx.outputs.html],
@@ -101,7 +100,12 @@ tensorboard_html_binary = rule(
         "input_path": attr.string(mandatory=True),
         "output_path": attr.string(mandatory=True),
         "data": attr.label_list(cfg="data", allow_files=True),
-        "deps": attr.label_list(aspects=[web_aspect], mandatory=True),
+        "deps": attr.label_list(
+            aspects=[
+                web_aspect,
+                legacy_js,
+            ],
+            mandatory=True),
         "external_assets": attr.string_dict(default={"/_/runfiles": "."}),
         "_jslibs": attr.label(
             default=Label("//tensorflow/tensorboard/java/org/tensorflow/tensorboard/vulcanize:jslibs"),