From e59b42f7d17375267200b273b4538f4b65d685fa Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 24 Mar 2020 13:18:05 +0100 Subject: [PATCH 1/3] add script to create Win VM --- tools/gce/create_win2019_container_vm.sh | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 tools/gce/create_win2019_container_vm.sh diff --git a/tools/gce/create_win2019_container_vm.sh b/tools/gce/create_win2019_container_vm.sh new file mode 100755 index 0000000000000..02db428ebb3a4 --- /dev/null +++ b/tools/gce/create_win2019_container_vm.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Creates a worker for debugging/experiments. +# The worker will have all the prerequisites that are installed on kokoro +# windows workers. + +set -ex + +cd "$(dirname "$0")" + +CLOUD_PROJECT=grpc-testing +ZONE=us-central1-b + +if [ "$1" != "" ] +then + INSTANCE_NAME="$1" +else + INSTANCE_NAME="${USER}-win2019-for-containers-test1" +fi + +MACHINE_TYPE=e2-standard-8 + +# The image version might need updating. +gcloud compute instances create "$INSTANCE_NAME" \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + --machine-type "$MACHINE_TYPE" \ + --boot-disk-size=400GB \ + --boot-disk-type pd-ssd \ + --image-project=windows-cloud \ + --image-family=windows-2019-for-containers + +# or use --image-family=windows-2019-core-for-containers From cdb3d823fdc1f554ff04c758d6b847f1b8a9a14e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Mar 2020 11:41:47 +0100 Subject: [PATCH 2/3] migrate windows RBE to win2019 pool --- WORKSPACE | 5 + third_party/toolchains/BUILD | 13 ++- third_party/toolchains/README.md | 5 + .../Dockerfile | 96 +++++++++---------- tools/remote_build/windows.bazelrc | 6 +- 5 files changed, 66 insertions(+), 59 deletions(-) create mode 100644 third_party/toolchains/README.md rename third_party/toolchains/{bazel_0.26.0_rbe_windows => rbe_win2019}/Dockerfile (64%) diff --git a/WORKSPACE b/WORKSPACE index f1eeb51414997..c3aa3b2aeccc5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -39,6 +39,11 @@ rbe_autoconfig( # that want to use other machines (such as LARGE_MACHINE) will override # this value. gce_machine_type = "n1-highmem-2", + # WARNING: the os_family constraint has only been introduced recently + # and older release branches select workers solely based on gce_machine_type. + # Worker pools needs to be configured with care to avoid accidentally running + # linux jobs on windows pool and vice versa (which would lead to a test breakage) + os_family = "Linux", ), # use exec_properties instead of deprecated remote_execution_properties use_legacy_platform_definition = False, diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD index a581d1fe1e3c0..d68f9f2c651bc 100644 --- a/third_party/toolchains/BUILD +++ b/third_party/toolchains/BUILD @@ -20,20 +20,23 @@ load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exe alias( name = "rbe_windows", - actual = ":rbe_windows_1803", + actual = ":rbe_windows_2019", ) # RBE Windows platform( - name = "rbe_windows_1803", + name = "rbe_windows_2019", constraint_values = [ "@bazel_tools//platforms:x86_64", "@bazel_tools//platforms:windows", - "@bazel_tools//tools/cpp:msvc", ], exec_properties = create_exec_properties_dict( - container_image = "docker://gcr.io/grpc-testing/rbe_windows_toolchain@sha256:75728e7d6d804090f71095e5fec627b18cfa1f47adc52096c209f2a687e06b2e", - gce_machine_type = "n1-highmem-2", + # See rbe_win2019/Dockerfile for image details + container_image = "docker://gcr.io/grpc-testing/rbe_windows2019_withdbg_toolchain@sha256:7b04ee7e29f942adbf4f70edd2ec4ba20a3e7237e1b54f5cae4b239c6ca41105", + + # Use a different machine type than used on linux to avoid accidentally scheduling linux jobs on windows workers and vice versa on older release branches + gce_machine_type = "n1-standard-2", os_family = "Windows", + labels = {"os": "windows_2019"}, ), ) diff --git a/third_party/toolchains/README.md b/third_party/toolchains/README.md new file mode 100644 index 0000000000000..4a26825646304 --- /dev/null +++ b/third_party/toolchains/README.md @@ -0,0 +1,5 @@ +# Windows RBE toolchain configuration + +- bazel_0.26.0_rbe_windows: The autogenerated toolchain for windows RBE. See go/rbe-windows-user-guide + +- rbe_win2019: The windows dockerfile to rebuild the docker image we use on RBE workers. See go/rbe-windows-user-guide \ No newline at end of file diff --git a/third_party/toolchains/bazel_0.26.0_rbe_windows/Dockerfile b/third_party/toolchains/rbe_win2019/Dockerfile similarity index 64% rename from third_party/toolchains/bazel_0.26.0_rbe_windows/Dockerfile rename to third_party/toolchains/rbe_win2019/Dockerfile index bbfce76436dbc..b6efe2fb0dc1f 100644 --- a/third_party/toolchains/bazel_0.26.0_rbe_windows/Dockerfile +++ b/third_party/toolchains/rbe_win2019/Dockerfile @@ -1,79 +1,76 @@ +# This dockerfile is taken from go/rbe-windows-user-guide +# (including the fix --compilation_mode=dbg) + # This Dockerfile creates an image that: # - Has the correct MTU setting for networking from inside the container to work. # - Has Visual Studio 2015 Build Tools installed. # - Has msys2 + git, curl, zip, unzip installed. # - Has Python 2.7 installed. -# - Has Bazel installed. - # TODO(jsharpe): Consider replacing "ADD $URI $DEST" with "Invoke-WebRequest -Method Get -Uri $URI -OutFile $DEST" - # Use the latest Windows Server Core image. # -# WARNING: What's the `:1803` about? There are two versions of Windows Server -# 2016: a "regular" one (corresponding to `microsoft/windowsservercore`) is on -# a slow release cadence, and a fast release cadence one (corresponding to -# `microsoft/windowsservercore:1803`). If you chose a different image than -# described above, probably omit the `:1803` or change it to a different -# number. -FROM microsoft/windowsservercore:1803 - +# WARNING: What's the `:ltsc2019` about? +# There are two versions of Windows Server 2019: +# 1. A "regular" one (corresponding to `mcr.microsoft.com/windows/servercore:ltsc2019`) +# is on a slow release cadence and is the Long-Term Servicing Channel. +# Mainstream support for this image will end on 1/9/2024. +# 2. A "fast" release cadence one (corresponding to +# `mcr.microsoft.com/windows/servercore:1909`) is the Semi-Annual Channel. +# Mainstream support for this image will end on 5/11/2021. +# +# If you choose a different +# image than described above, change the `:ltsc2019` tag. +# Start a temporary container in which we install 7-Zip to extract msys2 +FROM mcr.microsoft.com/windows/servercore:ltsc2019 as extract-msys2 SHELL ["powershell.exe", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; $VerbosePreference = 'Continue';"] - -# TODO(b/112379377): Workaround until bug is fixed. -RUN netsh interface ipv4 set subinterface \"vEthernet (Ethernet)\" mtu=1460 store=persistent - -# Install Visual Studio 2015 Build Tools. -RUN Invoke-WebRequest "https://download.microsoft.com/download/5/f/7/5f7acaeb-8363-451f-9425-68a90f98b238/visualcppbuildtools_full.exe" \ - -OutFile visualcppbuildtools_full.exe -UseBasicParsing ; \ - Start-Process -FilePath 'visualcppbuildtools_full.exe' -ArgumentList '/quiet', '/NoRestart' -Wait ; \ - Remove-Item .\visualcppbuildtools_full.exe; - -# TODO(jsharpe): Alternate install for msys2: https://github.com/StefanScherer/dockerfiles-windows/issues/30 - # Install 7-Zip and add it to the path. ADD https://www.7-zip.org/a/7z1801-x64.msi C:\\TEMP\\7z.msi RUN Start-Process msiexec.exe -ArgumentList \"/i C:\\TEMP\\7z.msi /qn /norestart /log C:\\TEMP\\7z_install_log.txt\" -wait RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \ $newpath = \"$oldpath;C:\Program Files\7-Zip\"; \ Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath - -# Install msys2, and add some extra tools. -ADD http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz C:\\TEMP\\msys2.tar.xz +# Extract msys2 +ADD http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20181211.tar.xz C:\\TEMP\\msys2.tar.xz RUN 7z x C:\TEMP\msys2.tar.xz -oC:\TEMP\msys2.tar RUN 7z x C:\TEMP\msys2.tar -oC:\tools +# Start building the actual image +FROM mcr.microsoft.com/windows/servercore:ltsc2019 +SHELL ["powershell.exe", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; $VerbosePreference = 'Continue';"] +# TODO(b/112379377): Workaround until bug is fixed. +RUN Get-NetAdapter | Where-Object Name -like "*Ethernet*" | ForEach-Object { & netsh interface ipv4 set subinterface $_.InterfaceIndex mtu=1460 store=persistent } +# Enable Long Paths for Win32 File/Folder APIs. +RUN New-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1 -PropertyType DWORD -Force + +# Install Visual Studio 2015 Build Tools. +RUN Invoke-WebRequest "https://download.microsoft.com/download/5/f/7/5f7acaeb-8363-451f-9425-68a90f98b238/visualcppbuildtools_full.exe" \ + -OutFile visualcppbuildtools_full.exe -UseBasicParsing ; \ + Start-Process -FilePath 'visualcppbuildtools_full.exe' -ArgumentList '/quiet', '/NoRestart', '/InstallSelectableItems "Win10SDK_VisibleV1"' -Wait ; \ + Remove-Item .\visualcppbuildtools_full.exe; +# Add ucrtbased.dll to the system directory to allow --compilation_mode=dbg to +# work. This DLL should be automatically copied to C:\Windows\System32 by the +# installer, but isn't when the installer is run on Docker, for some reason. +RUN Copy-Item \"C:\Program Files (x86)\Windows Kits\10\bin\x64\ucrt\ucrtbased.dll\" C:\Windows\System32 + + +# TODO(jsharpe): Alternate install for msys2: https://github.com/StefanScherer/dockerfiles-windows/issues/30 +# From the temporary extract-msys2 container, copy the tools directory to this container +COPY --from=extract-msys2 ["C:/tools", "C:/tools"] +# Add msys2 to the PATH variable RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \ $newpath = \"$oldpath;C:\tools\msys64;C:\tools\msys64\usr\bin\"; \ Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath -RUN Start-Process msys2 -ArgumentList 'pacman -noconfirm -Syuu git curl zip unzip' -Wait - +# Bazel documentation says to use -Syuu but this doesn't work in Docker. See +# http://g/foundry-windows/PDMVXbGew7Y +RUN bash.exe -c \"pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys && pacman --noconfirm -Syy git curl zip unzip\" # Install Visual C++ Redistributable for Visual Studio 2015: ADD https://download.microsoft.com/download/9/3/F/93FCF1E7-E6A4-478B-96E7-D4B285925B00/vc_redist.x64.exe C:\\TEMP\\vc_redist.x64.exe RUN C:\TEMP\vc_redist.x64.exe /quiet /install - # Install Python 2.7. ADD https://www.python.org/ftp/python/2.7.14/python-2.7.14.amd64.msi C:\\TEMP\\python.msi RUN Start-Process msiexec.exe -ArgumentList \"/i C:\\TEMP\\python.msi /qn /norestart /log C:\\TEMP\\python_install_log.txt\" -wait RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \ $newpath = \"$oldpath;C:\Python27\"; \ Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath - -# Install Bazel. -RUN Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name BAZEL_SH -Value \"C:\tools\msys64\usr\bin\bash.exe\" -RUN Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name BAZEL_VC -Value \"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\" -RUN [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12; -ADD https://github.com/bazelbuild/bazel/releases/download/0.26.0/bazel-0.26.0-windows-x86_64.exe C:\\bin\\bazel.exe -RUN $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path; \ - $newpath = \"$oldpath;C:\bin\"; \ - Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath - -# TODO(jsharpe): This requires entropy so may be problematic on a headless machine: https://wiki.archlinux.org/index.php/Pacman/Package_signing#Initializing_the_keyring -RUN Start-Process msys2 -ArgumentList 'pacman-key --init' -Wait -# TODO(jsharpe): If you don't run this then the next command can't succeed since it needs to prompt to remove catgets. -RUN pacman --noconfirm -R libcatgets catgets -# Bazel documentation says to use -Syuu but this doesn't work in Docker. See -# http://g/foundry-windows/PDMVXbGew7Y -RUN pacman --noconfirm -Syy git curl zip unzip - RUN \ Add-Type -AssemblyName \"System.IO.Compression.FileSystem\"; \ $zulu_url = \"https://cdn.azul.com/zulu/bin/zulu8.28.0.1-jdk8.0.163-win_x64.zip\"; \ @@ -88,9 +85,10 @@ RUN \ [Environment]::SetEnvironmentVariable(\"PATH\", $env:PATH, \"Machine\"); \ $env:JAVA_HOME = $zulu_root; \ [Environment]::SetEnvironmentVariable(\"JAVA_HOME\", $env:JAVA_HOME, \"Machine\") - # Restore default shell for Windows containers. SHELL ["cmd.exe", "/s", "/c"] - # Default to PowerShell if no other command specified. CMD ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] + + + diff --git a/tools/remote_build/windows.bazelrc b/tools/remote_build/windows.bazelrc index 4f6979478f9c1..c260670f0696e 100644 --- a/tools/remote_build/windows.bazelrc +++ b/tools/remote_build/windows.bazelrc @@ -1,9 +1,5 @@ startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 -# TODO(jtattermusch): Currently building with --compilation_mode=dbg is broken on Windows RBE -# so we need to explicitly override it to use fastbuild. -build:windows_dbg --compilation_mode=fastbuild - build --remote_cache=grpcs://remotebuildexecution.googleapis.com build --remote_executor=grpcs://remotebuildexecution.googleapis.com @@ -26,7 +22,7 @@ build --strategy=Closure=remote build --genrule_strategy=remote build --remote_timeout=7200 # very large value to avoid problems like https://github.com/grpc/grpc/issues/20777 -build --remote_instance_name=projects/grpc-testing/instances/grpc-windows-rbe-test +build --remote_instance_name=projects/grpc-testing/instances/default_instance build --verbose_failures=true From 2de0e4c7df39e14ec85bf30fa32700f84cf2fb83 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 25 Mar 2020 12:29:39 +0100 Subject: [PATCH 3/3] small fixup --- third_party/toolchains/BUILD | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD index d68f9f2c651bc..70b75900e6350 100644 --- a/third_party/toolchains/BUILD +++ b/third_party/toolchains/BUILD @@ -37,6 +37,7 @@ platform( # Use a different machine type than used on linux to avoid accidentally scheduling linux jobs on windows workers and vice versa on older release branches gce_machine_type = "n1-standard-2", os_family = "Windows", - labels = {"os": "windows_2019"}, + # labels only supported starting from https://github.com/bazelbuild/bazel-toolchains/pull/748 + #labels = {"os": "windows_2019"}, ), )