Merge pull request rstudio#12526 from rstudio/feature/sccache-support

Add sccache support

CMakeGlobals.txt

@@ -481,3 +481,7 @@ if(APPLE)
 
 endif()
 
+# If enabled, use caching for the build
+if(SCCACHE_ENABLED)
+ include(sccache)
+endif()

Jenkinsfile.pull-request

@@ -2,6 +2,7 @@ os = 'jammy'
 arch = 'amd64'
 flavor = 'server'
 type = 'DEB'
+AWS_ACCOUNT_ID = '749683154838'
 def utils
 
 pipeline {
@@ -42,14 +43,17 @@ pipeline {
  stages {
  stage('Prepare Build Container') {
  steps {
- pullBuildPush(
- image_name: 'jenkins/ide',
- image_tag: "${os}-${arch}-${env.CHANGE_TARGET.replaceAll('/', '-')}",
- dockerfile: "docker/jenkins/Dockerfile.${os}",
- build_arg_jenkins_uid: 'JENKINS_UID',
- build_arg_jenkins_gid: 'JENKINS_GID',
- builds_args: "--build-arg GITHUB_LOGIN=${GITHUB_LOGIN}",
- push: false)
+ // AWS is here for the S3 bucket that we use for sccache
+ withAWS(role: 'build', roleAccount: AWS_ACCOUNT_ID) {
+ pullBuildPush(
+ image_name: 'jenkins/ide',
+ image_tag: "${os}-${arch}-${env.CHANGE_TARGET.replaceAll('/', '-')}",
+ dockerfile: "docker/jenkins/Dockerfile.${os}",
+ build_arg_jenkins_uid: 'JENKINS_UID',
+ build_arg_jenkins_gid: 'JENKINS_GID',
+ builds_args: "--build-arg GITHUB_LOGIN=${GITHUB_LOGIN}",
+ push: false)
+ }
  }
  }
 
@@ -65,8 +69,11 @@ pipeline {
  stage('Build Package') {
  steps {
  dir ("package/linux") {
- // perform the compilation
- sh "PACKAGE_OS=${os} ./make-${flavor}-package ${type} clean"
+ // AWS is here for the S3 bucket that we use for sccache
+ withAWS(role: 'build', roleAccount: AWS_ACCOUNT_ID) {
+ // perform the compilation
+ sh "SCCACHE_ENABLED=1 PACKAGE_OS=${os} ./make-${flavor}-package ${type} clean"
+ }
  }
  }
  }

cmake/modules/sccache.cmake

@@ -0,0 +1,28 @@
+find_program(CCACHE_PROGRAM sccache)
+message(STATUS "Looking for SCCACHE...")
+
+if(CCACHE_PROGRAM)
+ message(STATUS "Found SCCACHE: ${CCACHE_PROGRAM}")
+ # Support Unix Makefiles and Ninja. See https://stackoverflow.com/a/24305849/1170370
+ # set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
+ set(CMAKE_C_COMPILER_LAUNCHER ${CCACHE_PROGRAM})
+ set(CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE_PROGRAM})
+endif()
+
+# ccache compatibility with XCode
+# see https://stackoverflow.com/a/36515503/1170370
+get_property(RULE_LAUNCH_COMPILE GLOBAL PROPERTY RULE_LAUNCH_COMPILE)
+if(RULE_LAUNCH_COMPILE AND CMAKE_GENERATOR STREQUAL "Xcode")
+ # Set up wrapper scripts
+ configure_file(launch-c.in launch-c)
+ configure_file(launch-cxx.in launch-cxx)
+ execute_process(COMMAND chmod a+rx
+ "${CMAKE_BINARY_DIR}/launch-c"
+ "${CMAKE_BINARY_DIR}/launch-cxx")
+
+ # Set Xcode project attributes to route compilation through our scripts
+ set(CMAKE_XCODE_ATTRIBUTE_CC "${CMAKE_BINARY_DIR}/launch-c")
+ set(CMAKE_XCODE_ATTRIBUTE_CXX "${CMAKE_BINARY_DIR}/launch-cxx")
+ set(CMAKE_XCODE_ATTRIBUTE_LD "${CMAKE_BINARY_DIR}/launch-c")
+ set(CMAKE_XCODE_ATTRIBUTE_LDPLUSPLUS "${CMAKE_BINARY_DIR}/launch-cxx")
+endif()

dependencies/common/install-common

@@ -29,6 +29,7 @@ set -e
 ./install-yaml-cpp
 ./install-crashpad
 ./install-panmirror
+./install-sccache
 
 if [ -e install-overlay ]
 then

dependencies/common/install-node

@@ -3,7 +3,7 @@
 #
 # install-node
 #
-# Copyright (C) 2022 by RStudio, Inc.
+# Copyright (C) 2022 Posit Software, PBC
 #
 # Unless you have received this program directly from Posit Software pursuant
 # to the terms of a commercial license agreement with Posit Software, then

dependencies/common/install-npm-dependencies

@@ -3,7 +3,7 @@
 #
 # install-npm-dependencies
 #
-# Copyright (C) 2022 by RStudio, Inc.
+# Copyright (C) 2022 Posit Software, PBC
 #
 # Unless you have received this program directly from Posit Software pursuant
 # to the terms of a commercial license agreement with Posit Software, then

dependencies/common/install-sccache

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+#
+# install-sccache
+#
+# Copyright (C) 2023 by Posit Software, PBC
+#
+# Unless you have received this program directly from RStudio pursuant
+# to the terms of a commercial license agreement with RStudio, then
+# this program is licensed to you under the terms of version 3 of the
+# GNU Affero General Public License. This program is distributed WITHOUT
+# ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
+# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
+# AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
+#
+#
+
+set -e
+
+source "$(dirname "${BASH_SOURCE[0]}")/../tools/rstudio-tools.sh"
+section "Installing sccache"
+
+# install dir
+export INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}"
+if ! [ -w "$INSTALL_DIR" ]; then
+ export INSTALL_DIR="$HOME/opt/bin"
+fi
+
+VERSION=0.3.1
+
+# default architecture to amd64/x86_64
+ARCH=x86_64
+SCCACHE_PACKAGE_HASH="94ea33aac8dcb358753f8240cc87345963cf83cda7c6af0395dff31ffdc88df4"
+
+# override to aarch64 if running on arm
+if [ "$(arch)" = "aarch64" ] ; then
+ ARCH=aarch64
+ SCCACHE_PACKAGE_HASH="1bf58385dc27b66324bb9ee82084e65c4d2e60baa19e3d16d2ab4da6c1ae66b2"
+fi
+
+URL=https://github.com/mozilla/sccache/releases/download/v${VERSION}/sccache-v${VERSION}-${ARCH}-unknown-linux-musl.tar.gz
+
+wget --progress=bar:force:noscroll ${URL} -O sccache.tar.gz \
+ && echo "${SCCACHE_PACKAGE_HASH} sccache.tar.gz" | sha256sum -c -
+
+tar -zxvpf sccache.tar.gz
+mkdir -p ${INSTALL_DIR}
+if [ ! -f ${INSTALL_DIR}/sccache ]; then
+ mv sccache-v${VERSION}-*/sccache ${INSTALL_DIR}/sccache
+
+ chmod +x ${INSTALL_DIR}/sccache
+ # this is so that boost/bjam will find and use sccache
+ ln -sf ${INSTALL_DIR}/sccache ${INSTALL_DIR}/ccache
+
+fi
+
+rm -rf sccache*

dependencies/common/install-yarn

@@ -3,7 +3,7 @@
 #
 # install-yarn
 #
-# Copyright (C) 2022 by RStudio, Inc.
+# Copyright (C) 2022 Posit Software, PBC
 #
 # Unless you have received this program directly from Posit Software pursuant
 # to the terms of a commercial license agreement with Posit Software, then

dependencies/common/install-yarn-linux-aarch64

@@ -3,7 +3,7 @@
 #
 # install-yarn-linux-aarch64
 #
-# Copyright (C) 2022 by RStudio, Inc.
+# Copyright (C) 2022 Posit Software, PBC
 #
 # Unless you have received this program directly from Posit Software pursuant
 # to the terms of a commercial license agreement with Posit Software, then

package/linux/make-package

@@ -225,6 +225,20 @@ if [ -n "${JENKINS_URL}" ]; then
  printenv
 fi
 
+# if SCCACHE_ENABLED environment variable is set, use sccache
+if [ -n "$SCCACHE_ENABLED" ]; then
+ echo "Using sccache"
+
+ if [ -n "$AWS_ACCESS_KEY_ID" ] || [ $(aws sts get-caller-identity --query "Account" --profile ${AWS_PROFILE:-sso}) -eq 14 ]; then
+ echo "AWS credentials valid, using S3 build cache"
+ export SCCACHE_BUCKET="rstudio-build-cache"
+ else
+ echo "No valid AWS SSO session, using only local build cache"
+ export SCCACHE_DIR=$(pwd)/object_file_cache
+ mkdir -p $SCCACHE_DIR
+ fi
+fi
+
 cmake -G"${CMAKE_GENERATOR}" \
  -DRSTUDIO_TARGET=$RSTUDIO_TARGET \
  -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
@@ -234,10 +248,15 @@ cmake -G"${CMAKE_GENERATOR}" \
  -DGWT_WWW_DIR="$BUILD_DIR/gwt/www" \
  -DGWT_EXTRAS_DIR="$BUILD_DIR/gwt/extras" \
  -DGWT_BUILD="$GWT_BUILD" \
+ -DSCCACHE_ENABLED=$SCCACHE_ENABLED \
  $PKG_DIR/../..
 
 cmake --build . --target all -- ${MAKEFLAGS}
 
+if [ -n "$SCCACHE_ENABLED" ]; then
+ sccache --show-stats
+fi
+
 if [ "$PACKAGE_TARGET" != "DEB" ]
 then
  fakeroot cpack --verbose --debug -G "$PACKAGE_TARGET"