ci: download Buildifier onto CI machines

Summary:
This adapts the existing Bazel download infrastructure to download the
stock `buildifier` binary. We don’t yet use it.

Test Plan:
Run on Travis; make sure that it installs properly.

wchargin-branch: buildifier-install

.travis.yml

@@ -18,6 +18,8 @@ env:
   global:
     - BAZEL=0.26.1
     - BAZEL_SHA256SUM=6c50e142a0a405d3d8598050d6c1b3920c8cdb82a7ffca6fc067cb474275148f
+    - BUILDIFIER=0.29.0
+    - BUILDIFIER_SHA256SUM=4c985c883eafdde9c0e8cf3c8595b8bfdf32e77571c369bf8ddae83b042028d6
   matrix:
     - TF_VERSION_ID=tensorflow==1.15.0rc3
     - TF_VERSION_ID=tf-nightly==2.1.0.dev20191118
@@ -41,6 +43,8 @@ before_install:
   - elapsed "before_install"
   - ci/download_bazel.sh "${BAZEL}" "${BAZEL_SHA256SUM}" ~/bazel
   - sudo mv ~/bazel /usr/local/bin/bazel
+  - ci/download_buildifier.sh "${BUILDIFIER}" "${BUILDIFIER_SHA256SUM}" ~/buildifier
+  - sudo mv ~/buildifier /usr/local/bin/buildifier
   - cp ci/bazelrc ~/.bazelrc
   - elapsed "before_install (done)"
 
@@ -80,6 +84,8 @@ before_script:
   - if [ -n "${PY3}" ]; then black --check .; fi
   # Lint .yaml docs files. Use '# yamllint disable-line rule:foo' to suppress.
   - yamllint -c docs/.yamllint docs docs/.yamllint
+  # TODO(#2963): Lint BUILD files
+  - buildifier --version
   # Make sure that IPython notebooks have valid Markdown.
   - tensorboard/tools/docs_list_format_test.sh
   # Make sure we aren't accidentally including work-in-progress code.

ci/download_bazel.sh

@@ -31,19 +31,8 @@ version="$1"
 checksum="$2"
 dest="$3"
 
-temp_dest="$(mktemp)"
-
 mirror_url="http://mirror.tensorflow.org/github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-linux-x86_64"
 github_url="https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-linux-x86_64"
 
-for url in "${mirror_url}" "${github_url}"; do
-  wget -t 3 -O "${temp_dest}" "${url}" \
-    && printf "%s  %s\n" "${checksum}" "${temp_dest}" | shasum -a 256 --check \
-    || { rm -f "${temp_dest}"; continue; }
-  mv "${temp_dest}" "${dest}"
-  break
-done
-
-[ -f "${dest}" ]
-chmod +x "${dest}"
-ls -l "${dest}"
+exec "$(dirname "$0")/download_executable.sh" "${checksum}" "${dest}" \
+    "${mirror_url}" "${github_url}"

ci/download_buildifier.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Script to download Buildifier binary directly onto a build machine.
+
+set -e
+
+die() {
+  printf >&2 "%s\n" "$1"
+  exit 1
+}
+
+if [ "$#" -ne 3 ]; then
+  die "Usage: $0 <version> <sha256sum> <destination-file>"
+fi
+
+version="$1"
+checksum="$2"
+dest="$3"
+
+mirror_url="http://mirror.tensorflow.org/github.com/bazelbuild/buildtools/releases/download/${version}/buildifier"
+github_url="https://github.com/bazelbuild/buildtools/releases/download/${version}/buildifier"
+
+exec "$(dirname "$0")/download_executable.sh" "${checksum}" "${dest}" \
+    "${mirror_url}" "${github_url}"

ci/download_executable.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Script to download a binary directly onto a build machine, with
+# checksum verification.
+
+set -e
+
+die() {
+  printf >&2 "%s\n" "$1"
+  exit 1
+}
+
+if [ "$#" -lt 3 ]; then
+  die "Usage: $0 <sha256sum> <destination-file> <url> [<url>...]"
+fi
+
+checksum="$1"
+dest="$2"
+shift 2
+
+temp_dest="$(mktemp)"
+
+for url; do
+  wget -t 3 -O "${temp_dest}" "${url}" \
+    && printf "%s  %s\n" "${checksum}" "${temp_dest}" | shasum -a 256 --check \
+    || { rm -f "${temp_dest}"; continue; }
+  mv "${temp_dest}" "${dest}"
+  break
+done
+
+
+[ -f "${dest}" ]
+chmod +x "${dest}"
+ls -l "${dest}"

tensorboard/tools/mirror_urls_test.sh

@@ -42,13 +42,13 @@ check_urls_resolve() {
     # shellcheck disable=SC2016
     check_cmd='curl -sfL "$1" >/dev/null || printf "%s\n" "$1"'
     url_pcre='(?<=")https?://mirror\.tensorflow\.org/[^"]*'
-    exclude=':!ci/download_bazel.sh'  # uses a '${version}' format string
+    exclude_bazel=':!ci/download_bazel.sh'  # uses a '${version}' format string
+    exclude_buildifier=':!ci/download_buildifier.sh'  # likewise
     # We use `git-grep` to efficiently get an initial result set, then
     # filter it down with GNU `grep` separately, because `git-grep` only
     # learned `-o` in Git v2.19; Travis uses v2.15.1.
-    git grep -Ph "${url_pcre}" "${exclude}" \
+    git grep -Ph "${url_pcre}" "${exclude_bazel}" "${exclude_buildifier}" \
         | grep -o 'https\?://mirror\.tensorflow\.org/[^"]*' \
-        | grep -vF -- "${exclude}" \
         | sort \
         | uniq \
         | xargs -n 1 -P 32 -- sh -c "${check_cmd}" unused \