Merge pull request grpc#22473 from jtattermusch/win2019_bazel_rbe

Migrate windows RBE to win2019 pool

WORKSPACE

@@ -39,6 +39,11 @@ rbe_autoconfig(
         # that want to use other machines (such as LARGE_MACHINE) will override
         # this value.
         gce_machine_type = "n1-highmem-2",
+        # WARNING: the os_family constraint has only been introduced recently
+        # and older release branches select workers solely based on gce_machine_type.
+        # Worker pools needs to be configured with care to avoid accidentally running
+        # linux jobs on windows pool and vice versa (which would lead to a test breakage)
+        os_family = "Linux",
     ),
     # use exec_properties instead of deprecated remote_execution_properties
     use_legacy_platform_definition = False,

third_party/toolchains/BUILD

@@ -20,20 +20,24 @@ load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exe
 
 alias(
     name = "rbe_windows",
-    actual = ":rbe_windows_1803",
+    actual = ":rbe_windows_2019",
 )
 
 # RBE Windows
 platform(
-    name = "rbe_windows_1803",
+    name = "rbe_windows_2019",
     constraint_values = [
         "@bazel_tools//platforms:x86_64",
         "@bazel_tools//platforms:windows",
-        "@bazel_tools//tools/cpp:msvc",
     ],
     exec_properties = create_exec_properties_dict(
-        container_image = "docker://gcr.io/grpc-testing/rbe_windows_toolchain@sha256:75728e7d6d804090f71095e5fec627b18cfa1f47adc52096c209f2a687e06b2e",
-        gce_machine_type = "n1-highmem-2",
+        # See rbe_win2019/Dockerfile for image details
+        container_image = "docker://gcr.io/grpc-testing/rbe_windows2019_withdbg_toolchain@sha256:7b04ee7e29f942adbf4f70edd2ec4ba20a3e7237e1b54f5cae4b239c6ca41105",
+
+        # Use a different machine type than used on linux to avoid accidentally scheduling linux jobs on windows workers and vice versa on older release branches
+        gce_machine_type = "n1-standard-2",
         os_family = "Windows",
+        # labels only supported starting from https://github.com/bazelbuild/bazel-toolchains/pull/748
+        #labels = {"os": "windows_2019"},
     ),
 )

third_party/toolchains/README.md

@@ -0,0 +1,5 @@
+# Windows RBE toolchain configuration
+
+- bazel_0.26.0_rbe_windows: The autogenerated toolchain for windows RBE. See go/rbe-windows-user-guide
+
+- rbe_win2019: The windows dockerfile to rebuild the docker image we use on RBE workers. See go/rbe-windows-user-guide

third_party/toolchains/bazel_0.26.0_rbe_windows/Dockerfile → third_party/toolchains/rbe_win2019/Dockerfile

@@ -1,79 +1,76 @@
+# This dockerfile is taken from go/rbe-windows-user-guide
+# (including the fix --compilation_mode=dbg)
+
 # This Dockerfile creates an image that:
 # - Has the correct MTU setting for networking from inside the container to work.
 # - Has Visual Studio 2015 Build Tools installed.
 # - Has msys2 + git, curl, zip, unzip installed.
 # - Has Python 2.7 installed.
-# - Has Bazel installed.
-
 # TODO(jsharpe): Consider replacing "ADD $URI $DEST" with "Invoke-WebRequest -Method Get -Uri $URI -OutFile $DEST"
-
 # Use the latest Windows Server Core image.
 #
-# WARNING: What's the `:1803` about? There are two versions of Windows Server
-# 2016: a "regular" one (corresponding to `microsoft/windowsservercore`) is on
-# a slow release cadence, and a fast release cadence one (corresponding to
-# `microsoft/windowsservercore:1803`). If you chose a different image than
-# described above, probably omit the `:1803` or change it to a different
-# number.
-FROM microsoft/windowsservercore:1803
-
+# WARNING: What's the `:ltsc2019` about?
+# There are two versions of Windows Server 2019:
+# 1. A "regular" one (corresponding to `mcr.microsoft.com/windows/servercore:ltsc2019`)
+# is on a slow release cadence and is the Long-Term Servicing Channel.
+# Mainstream support for this image will end on 1/9/2024.
+# 2. A "fast" release cadence one (corresponding to
+# `mcr.microsoft.com/windows/servercore:1909`) is the Semi-Annual Channel.
+# Mainstream support for this image will end on 5/11/2021.
+#
+# If you choose a different
+# image than described above, change the `:ltsc2019` tag.
+# Start a temporary container in which we install 7-Zip to extract msys2
+FROM mcr.microsoft.com/windows/servercore:ltsc2019 as extract-msys2
 SHELL ["powershell.exe", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; $VerbosePreference = 'Continue';"]
-
-# TODO(b/112379377): Workaround until bug is fixed.
-RUN netsh interface ipv4 set subinterface \"vEthernet (Ethernet)\" mtu=1460 store=persistent
-
-# Install Visual Studio 2015 Build Tools.
-RUN Invoke-WebRequest "https://download.microsoft.com/download/5/f/7/5f7acaeb-8363-451f-9425-68a90f98b238/visualcppbuildtools_full.exe" \
-                -OutFile visualcppbuildtools_full.exe -UseBasicParsing ; \
-        Start-Process -FilePath 'visualcppbuildtools_full.exe' -ArgumentList '/quiet', '/NoRestart' -Wait ; \
-        Remove-Item .\visualcppbuildtools_full.exe;
-
-# TODO(jsharpe): Alternate install for msys2: https://github.com/StefanScherer/dockerfiles-windows/issues/30
-
 # Install 7-Zip and add it to the path.
 ADD https://www.7-zip.org/a/7z1801-x64.msi C:\\TEMP\\7z.msi
 RUN Start-Process msiexec.exe -ArgumentList \"/i C:\\TEMP\\7z.msi /qn /norestart /log C:\\TEMP\\7z_install_log.txt\" -wait
 RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \
   $newpath = \"$oldpath;C:\Program Files\7-Zip\"; \
   Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath
-
-# Install msys2, and add some extra tools.
-ADD http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz C:\\TEMP\\msys2.tar.xz
+# Extract msys2
+ADD http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20181211.tar.xz C:\\TEMP\\msys2.tar.xz
 RUN 7z x C:\TEMP\msys2.tar.xz -oC:\TEMP\msys2.tar
 RUN 7z x C:\TEMP\msys2.tar -oC:\tools
+# Start building the actual image
+FROM mcr.microsoft.com/windows/servercore:ltsc2019
+SHELL ["powershell.exe", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; $VerbosePreference = 'Continue';"]
+# TODO(b/112379377): Workaround until bug is fixed.
+RUN Get-NetAdapter | Where-Object Name -like "*Ethernet*" | ForEach-Object { & netsh interface ipv4 set subinterface $_.InterfaceIndex mtu=1460 store=persistent }
+# Enable Long Paths for Win32 File/Folder APIs.
+RUN New-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1 -PropertyType DWORD -Force
+
+# Install Visual Studio 2015 Build Tools.
+RUN Invoke-WebRequest "https://download.microsoft.com/download/5/f/7/5f7acaeb-8363-451f-9425-68a90f98b238/visualcppbuildtools_full.exe" \
+                -OutFile visualcppbuildtools_full.exe -UseBasicParsing ; \
+        Start-Process -FilePath 'visualcppbuildtools_full.exe' -ArgumentList '/quiet', '/NoRestart', '/InstallSelectableItems "Win10SDK_VisibleV1"' -Wait ; \
+        Remove-Item .\visualcppbuildtools_full.exe;
+# Add ucrtbased.dll to the system directory to allow --compilation_mode=dbg to
+# work. This DLL should be automatically copied to C:\Windows\System32 by the
+# installer, but isn't when the installer is run on Docker, for some reason.
+RUN Copy-Item \"C:\Program Files (x86)\Windows Kits\10\bin\x64\ucrt\ucrtbased.dll\" C:\Windows\System32
+
+
+# TODO(jsharpe): Alternate install for msys2: https://github.com/StefanScherer/dockerfiles-windows/issues/30
+# From the temporary extract-msys2 container, copy the tools directory to this container
+COPY --from=extract-msys2 ["C:/tools", "C:/tools"]
+# Add msys2 to the PATH variable
 RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \
   $newpath = \"$oldpath;C:\tools\msys64;C:\tools\msys64\usr\bin\"; \
   Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath
-RUN Start-Process msys2 -ArgumentList 'pacman -noconfirm -Syuu git curl zip unzip' -Wait
-
+# Bazel documentation says to use -Syuu but this doesn't work in Docker. See
+# http://g/foundry-windows/PDMVXbGew7Y
+RUN bash.exe -c \"pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys && pacman --noconfirm -Syy git curl zip unzip\"
 # Install Visual C++ Redistributable for Visual Studio 2015:
 ADD https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe C:\\TEMP\\vc_redist.x64.exe
 RUN C:\TEMP\vc_redist.x64.exe /quiet /install
-
 # Install Python 2.7.
 ADD https://www.python.org/ftp/python/2.7.14/python-2.7.14.amd64.msi C:\\TEMP\\python.msi
 RUN Start-Process msiexec.exe -ArgumentList \"/i C:\\TEMP\\python.msi /qn /norestart /log C:\\TEMP\\python_install_log.txt\" -wait
 RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \
   $newpath = \"$oldpath;C:\Python27\"; \
   Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath
-
-# Install Bazel.
-RUN Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name BAZEL_SH -Value \"C:\tools\msys64\usr\bin\bash.exe\"
-RUN Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name BAZEL_VC -Value \"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\"
-RUN [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12;
-ADD https://github.com/bazelbuild/bazel/releases/download/0.26.0/bazel-0.26.0-windows-x86_64.exe C:\\bin\\bazel.exe
-RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \
-  $newpath = \"$oldpath;C:\bin\"; \
-  Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath
-
-# TODO(jsharpe): This requires entropy so may be problematic on a headless machine: https://wiki.archlinux.org/index.php/Pacman/Package_signing#Initializing_the_keyring
-RUN Start-Process msys2 -ArgumentList 'pacman-key --init' -Wait
-# TODO(jsharpe): If you don't run this then the next command can't succeed since it needs to prompt to remove catgets.
-RUN pacman --noconfirm -R libcatgets catgets
-# Bazel documentation says to use -Syuu but this doesn't work in Docker. See
-# http://g/foundry-windows/PDMVXbGew7Y
-RUN pacman --noconfirm -Syy git curl zip unzip
-
 RUN \
   Add-Type -AssemblyName \"System.IO.Compression.FileSystem\"; \
   $zulu_url = \"https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-win_x64.zip\"; \
@@ -88,9 +85,10 @@ RUN \
   [Environment]::SetEnvironmentVariable(\"PATH\", $env:PATH, \"Machine\"); \
   $env:JAVA_HOME = $zulu_root; \
   [Environment]::SetEnvironmentVariable(\"JAVA_HOME\", $env:JAVA_HOME, \"Machine\")
-
 # Restore default shell for Windows containers.
 SHELL ["cmd.exe", "/s", "/c"]
-
 # Default to PowerShell if no other command specified.
 CMD ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"]
+
+
+

tools/gce/create_win2019_container_vm.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Creates a worker for debugging/experiments.
+# The worker will have all the prerequisites that are installed on kokoro
+# windows workers.
+
+set -ex
+
+cd "$(dirname "$0")"
+
+CLOUD_PROJECT=grpc-testing
+ZONE=us-central1-b
+
+if [ "$1" != "" ]
+then
+  INSTANCE_NAME="$1"
+else
+  INSTANCE_NAME="${USER}-win2019-for-containers-test1"
+fi
+
+MACHINE_TYPE=e2-standard-8
+
+# The image version might need updating.
+gcloud compute instances create "$INSTANCE_NAME" \
+    --project="$CLOUD_PROJECT" \
+    --zone "$ZONE" \
+    --machine-type "$MACHINE_TYPE" \
+    --boot-disk-size=400GB \
+    --boot-disk-type pd-ssd \
+    --image-project=windows-cloud \
+    --image-family=windows-2019-for-containers
+
+# or use --image-family=windows-2019-core-for-containers

tools/remote_build/windows.bazelrc

@@ -1,9 +1,5 @@
 startup --host_jvm_args=-Dbazel.DigestFunction=SHA256
 
-# TODO(jtattermusch): Currently building with --compilation_mode=dbg is broken on Windows RBE
-# so we need to explicitly override it to use fastbuild.
-build:windows_dbg --compilation_mode=fastbuild
-
 build --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build --remote_executor=grpcs://remotebuildexecution.googleapis.com
 
@@ -26,7 +22,7 @@ build --strategy=Closure=remote
 build --genrule_strategy=remote
 build --remote_timeout=7200  # very large value to avoid problems like https://github.com/grpc/grpc/issues/20777
 
-build --remote_instance_name=projects/grpc-testing/instances/grpc-windows-rbe-test
+build --remote_instance_name=projects/grpc-testing/instances/default_instance
 
 build --verbose_failures=true