Skip to content

Commit

Permalink
Merge pull request dotnet#20508 from weshaggard/UpdateCI
Browse files Browse the repository at this point in the history
Clean out portable from CI scripts and add OSX
  • Loading branch information
weshaggard authored Jun 19, 2017
2 parents 4a9c4c8 + bbd891f commit e1fe61f
Show file tree
Hide file tree
Showing 6 changed files with 223 additions and 207 deletions.
25 changes: 13 additions & 12 deletions buildpipeline/portable-linux.groovy → buildpipeline/linux.groovy
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
@Library('dotnet-ci') _

// Incoming parameters. Access with "params.<param name>".
// Config - Build configuration. Note that we don't using 'Configuration' since it's used
// in the build scripts and this can cause problems.
// OuterLoop - If true, runs outerloop, if false runs just innerloop
// Note that the parameters will be set as env variables so we cannot use names that conflict
// with the engineering system parameter names.
// CGroup - Build configuration.
// TestOuter - If true, runs outerloop, if false runs just innerloop

def submittedHelixJson = null

Expand Down Expand Up @@ -35,14 +36,14 @@ simpleDockerNode('microsoft/dotnet-buildtools-prereqs:rhel7_prereqs_2') {
sh "./sync.sh -p -- /p:ArchGroup=x64"
}
stage ('Build Product') {
sh "./build.sh -buildArch=x64 -${params.Config}"
sh "./build.sh -buildArch=x64 -${params.CGroup}"
}
stage ('Build Tests') {
def additionalArgs = ''
if (params.OuterLoop) {
if (params.TestOuter) {
additionalArgs = '-Outerloop'
}
sh "./build-tests.sh -buildArch=x64 -${params.Config} -SkipTests ${additionalArgs} -- /p:ArchiveTests=true /p:EnableDumpling=true"
sh "./build-tests.sh -buildArch=x64 -${params.CGroup} -SkipTests ${additionalArgs} -- /p:ArchiveTests=true /p:EnableDumpling=true"
}
stage ('Submit To Helix For Testing') {
// Bind the credentials
Expand All @@ -63,14 +64,14 @@ simpleDockerNode('microsoft/dotnet-buildtools-prereqs:rhel7_prereqs_2') {
'Ubuntu.1610.Amd64.Open',
'suse.422.amd64.Open',
'fedora.25.amd64.Open',]
if (params.OuterLoop) {
if (params.TestOuter) {
targetHelixQueues += ['Debian.90.Amd64.Open',
'Fedora.26.Amd64.Open',
'SLES.12.Amd64.Open',
'Ubuntu.1704.Amd64.Open',]
}
}

sh "./Tools/msbuild.sh src/upload-tests.proj /p:ArchGroup=x64 /p:ConfigurationGroup=${params.Config} /p:TestProduct=corefx /p:TimeoutInSeconds=1200 /p:TargetOS=Linux /p:HelixJobType=test/functional/cli/ /p:HelixSource=${helixSource} /p:BuildMoniker=${helixBuild} /p:HelixCreator=${helixCreator} /p:CloudDropAccountName=dotnetbuilddrops /p:CloudResultsAccountName=dotnetjobresults /p:CloudDropAccessToken=\$CloudDropAccessToken /p:CloudResultsAccessToken=\$OutputCloudResultsAccessToken /p:HelixApiEndpoint=https://helix.dot.net/api/2017-04-14/jobs /p:TargetQueues=${targetHelixQueues.join('+')} /p:HelixLogFolder=${WORKSPACE}/${logFolder}/ /p:HelixCorrelationInfoFileName=SubmittedHelixRuns.txt"
sh "./Tools/msbuild.sh src/upload-tests.proj /p:ArchGroup=x64 /p:ConfigurationGroup=${params.CGroup} /p:TestProduct=corefx /p:TimeoutInSeconds=1200 /p:TargetOS=Linux /p:HelixJobType=test/functional/cli/ /p:HelixSource=${helixSource} /p:BuildMoniker=${helixBuild} /p:HelixCreator=${helixCreator} /p:CloudDropAccountName=dotnetbuilddrops /p:CloudResultsAccountName=dotnetjobresults /p:CloudDropAccessToken=\$CloudDropAccessToken /p:CloudResultsAccessToken=\$OutputCloudResultsAccessToken /p:HelixApiEndpoint=https://helix.dot.net/api/2017-04-14/jobs /p:TargetQueues=${targetHelixQueues.join('+')} /p:HelixLogFolder=${WORKSPACE}/${logFolder}/ /p:HelixCorrelationInfoFileName=SubmittedHelixRuns.txt"

submittedHelixJson = readJSON file: "${logFolder}/SubmittedHelixRuns.txt"
}
Expand All @@ -79,11 +80,11 @@ simpleDockerNode('microsoft/dotnet-buildtools-prereqs:rhel7_prereqs_2') {

stage ('Execute Tests') {
def contextBase
if (params.OuterLoop) {
contextBase = "Linux x64 Tests w/outer - ${params.Config}"
if (params.TestOuter) {
contextBase = "Linux x64 Tests w/outer - ${params.CGroup}"
}
else {
contextBase = "Linux x64 Tests - ${params.Config}"
contextBase = "Linux x64 Tests - ${params.CGroup}"
}
waitForHelixRuns(submittedHelixJson, contextBase)
}
49 changes: 49 additions & 0 deletions buildpipeline/osx.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
@Library('dotnet-ci') _

// Incoming parameters. Access with "params.<param name>".
// Note that the parameters will be set as env variables so we cannot use names that conflict
// with the engineering system parameter names.
// CGroup - Build configuration.
// TestOuter - If true, runs outerloop, if false runs just innerloop

def submittedHelixJson = null

simpleNode('OSX10.12','latest') {
stage ('Checkout source') {
checkout scm
}

def logFolder = getLogFolder()

stage ('Initialize tools') {
try {
// Workaround nuget issue https://github.com/NuGet/Home/issues/5085 were we need to set HOME
// Init tools
sh 'HOME=\$WORKSPACE/tempHome ./init-tools.sh'
}
catch (err) {
// On errors for build tools initializations, it's useful to echo the contents of the file
// for easy diagnosis. This could also be copied to the log directory
sh 'cat init-tools.log'
// Ensure the build result is still propagated.
throw err
}
}
stage ('Generate version assets') {
// Generate the version assets. Do we need to even do this for non-official builds?
sh "./build-managed.sh -- /t:GenerateVersionSourceFile /p:GenerateVersionSourceFile=true"
}
stage ('Sync') {
sh "HOME=\$WORKSPACE/tempHome ./sync.sh -p -- /p:ArchGroup=x64"
}
stage ('Build Product') {
sh "HOME=\$WORKSPACE/tempHome ./build.sh -buildArch=x64 -${params.CGroup}"
}
stage ('Build Tests') {
def additionalArgs = ''
if (params.TestOuter) {
additionalArgs = '-Outerloop'
}
sh "HOME=\$WORKSPACE/tempHome ./build-tests.sh -buildArch=x64 -${params.CGroup} ${additionalArgs} -- /p:ArchiveTests=true /p:EnableDumpling=true"
}
}
76 changes: 43 additions & 33 deletions buildpipeline/pipelinejobs.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -10,40 +10,50 @@ def project = GithubProject
def branch = GithubBranchName

// **************************
// Define innerloop testing. These jobs run on every merge and a subset of them run on every PR, the ones
// that don't run per PR can be requested via a magic phrase.
// Define innerloop testing. Any configuration in ForPR will run for every PR but all other configurations
// will have a trigger that can be
// **************************
def linuxPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/portable-linux.groovy')

['netcoreapp'].each { targetGroup ->
['Debug', 'Release'].each { configurationGroup ->
['Linux x64'].each { osName ->
// Runs the portable-linux.groovy pipeline on the target Helix queues mentioned in the pipeline. Currently:
// CentOS 7.3, RedHat 7.3, Debian 8.7, Ubuntu 14.04, Ubuntu 16.04, Ubuntu 16.10, openSuSE 42.2 and Fedora 25

// One for just innerloop.
linuxPipeline.triggerPipelineOnEveryGithubPR("Portable ${osName} ${configurationGroup} Build", ['Config':configurationGroup, 'OuterLoop':false])
// Add one for outerloop
linuxPipeline.triggerPipelineOnGithubPRComment("Portable Outerloop ${osName} ${configurationGroup} Build", ['Config':configurationGroup, 'OuterLoop':true])
}
}
}

// Create a pipeline for portable windows
def windowsPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/portable-windows.groovy')
['netcoreapp'].each { targetGroup ->
['Debug', 'Release'].each { configurationGroup ->
['Windows x64'].each { osName ->
// Runs the portable-windows.groovy pipeline on the target Helix queues mentioned in the pipeline. Currently:
// Windows 10, Windows 7, Windows 8.1 and Windows Nano

// One for just innerloop
windowsPipeline.triggerPipelineOnEveryGithubPR("Portable ${osName} ${configurationGroup} Build", ['Config':configurationGroup, 'OuterLoop':false])
// Add one for outerloop
windowsPipeline.triggerPipelineOnGithubPRComment("Portable Outerloop ${osName} ${configurationGroup} Build", ['Config':configurationGroup, 'OuterLoop':true])
}
}
}

def linPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/linux.groovy')
def osxPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/osx.groovy')
def winPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/windows.groovy')

def configurations = [
['TGroup':"netcoreapp", 'Pipeline':linPipeline, 'Name':'Linux' ,'ForPR':"Release-x64", 'Arch':['x64']],
['TGroup':"netcoreapp", 'Pipeline':osxPipeline, 'Name':'OSX', 'ForPR':"Debug-x64", 'Arch':['x64']],
['TGroup':"netcoreapp", 'Pipeline':winPipeline, 'Name':'Windows' , 'ForPR':"Debug-x64|Release-x86"],
['TGroup':"netfx", 'Pipeline':winPipeline, 'Name':'NETFX', 'ForPR':"Release-x86"],
['TGroup':"uap", 'Pipeline':winPipeline, 'Name':'UWP CoreCLR', 'ForPR':"Debug-x64"],
['TGroup':"uapaot", 'Pipeline':winPipeline, 'Name':'UWP NETNative', 'ForPR':"Release-x86"],
['TGroup':"all", 'Pipeline':winPipeline, 'Name':'Packaging All Configurations', 'ForPR':"Debug-x64"],
]

configurations.each { config ->
['Debug', 'Release'].each { configurationGroup ->
(config.Arch ?: ['x64', 'x86']).each { archGroup ->
def triggerName = "${config.Name} ${archGroup} ${configurationGroup} Build"

def pipeline = config.Pipeline
def params = ['TGroup':config.TGroup,
'CGroup':configurationGroup,
'AGroup':archGroup,
'TestOuter': false]

// Add default PR triggers for particular configurations but manual triggers for all
if (config.ForPR.contains("${configurationGroup}-${archGroup}")) {
pipeline.triggerPipelineOnEveryGithubPR(triggerName, params)
}
else {
pipeline.triggerPipelineOnGithubPRComment(triggerName, params)
}

// Add trigger for all configurations to run on merge
pipeline.triggerPipelineOnGithubPush(params)

// Add optional PR trigger for Outerloop test runs
params.TestOuter = true
pipeline.triggerPipelineOnGithubPRComment("Outerloop ${triggerName}", params)
}}}

JobReport.Report.generateJobReport(out)

Expand Down
71 changes: 0 additions & 71 deletions buildpipeline/portable-windows.groovy

This file was deleted.

107 changes: 107 additions & 0 deletions buildpipeline/windows.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
@Library('dotnet-ci') _

// Incoming parameters. Access with "params.<param name>".
// Note that the parameters will be set as env variables so we cannot use names that conflict
// with the engineering system parameter names.
// TGroup - The target framework to build.
// CGroup - Build configuration.
// TestOuter - If true, runs outerloop, if false runs just innerloop

def submittedHelixJson = null
def submitToHelix = (params.TGroup == 'netcoreapp')

simpleNode('Windows_NT','latest') {
stage ('Checkout source') {
retry (10) {
checkout scm
}
}

def logFolder = getLogFolder()
def framework = ''
if (params.TGroup == 'all') {
framework = '-allConfigurations'
}
else {
framework = "-framework:${params.TGroup}"
}
def buildTests = (params.TGroup != 'all')

stage ('Initialize tools') {
try {
// Init tools
bat '.\\init-tools.cmd'
// Temporarily always dump the init-tools.log to try and identify why sometimes it takes really long
bat 'type init-tools.log'
}
catch (err) {
// On errors for build tools initializations, it's useful to echo the contents of the file
// for easy diagnosis. This could also be copied to the log directory
bat 'type init-tools.log'
// Ensure the build result is still propagated.
throw err
}
}
stage ('Sync') {
bat ".\\sync.cmd -p -- /p:ArchGroup=${params.AGroup} /p:RuntimeOS=win10"
}
stage ('Generate Version Assets') {
bat '.\\build-managed.cmd -GenerateVersion'
}
stage ('Build Product') {
bat ".\\build.cmd ${framework} -buildArch=${params.AGroup} -${params.CGroup} -- /p:RuntimeOS=win10"
}
if (buildTests) {
stage ('Build Tests') {
def additionalArgs = ''
def archiveTests = 'false'
if (params.TestOuter) {
additionalArgs += ' -Outerloop'
}
if (submitToHelix) {
archiveTests = 'true'
}
if (submitToHelix || params.TGroup == 'uap' || params.TGroup == 'uapaot') {
additionalArgs += ' -SkipTests'
}
bat ".\\build-tests.cmd ${framework} -buildArch=${params.AGroup} -${params.CGroup}${additionalArgs} -- /p:RuntimeOS=win10 /p:ArchiveTests=${archiveTests}"
}
}
if (submitToHelix) {
stage ('Submit To Helix For Testing') {
// Bind the credentials
withCredentials([string(credentialsId: 'CloudDropAccessToken', variable: 'CloudDropAccessToken'),
string(credentialsId: 'OutputCloudResultsAccessToken', variable: 'OutputCloudResultsAccessToken')]) {
// Ask the CI SDK for a Helix source that makes sense. This ensures that this pipeline works for both PR and non-PR cases
def helixSource = getHelixSource()
// Ask the CI SDK for a Build that makes sense. We currently use the hash for the build
def helixBuild = getCommit()
// Get the user that should be associated with the submission
def helixCreator = getUser()

// Target queues
def targetHelixQueues = ['Windows.10.Amd64.Open',
'Windows.10.Nano.Amd64.Open',
'Windows.7.Amd64.Open',
'Windows.81.Amd64.Open']

bat "\"%VS140COMNTOOLS%\\VsDevCmd.bat\" && msbuild src\\upload-tests.proj /p:ArchGroup=${params.AGroup} /p:ConfigurationGroup=${params.CGroup} /p:TestProduct=corefx /p:TimeoutInSeconds=1200 /p:TargetOS=Windows_NT /p:HelixJobType=test/functional/cli/ /p:HelixSource=${helixSource} /p:BuildMoniker=${helixBuild} /p:HelixCreator=${helixCreator} /p:CloudDropAccountName=dotnetbuilddrops /p:CloudResultsAccountName=dotnetjobresults /p:CloudDropAccessToken=%CloudDropAccessToken% /p:CloudResultsAccessToken=%OutputCloudResultsAccessToken% /p:HelixApiEndpoint=https://helix.dot.net/api/2017-04-14/jobs /p:TargetQueues=\"${targetHelixQueues.join(',')}\" /p:HelixLogFolder= /p:HelixLogFolder=${WORKSPACE}\\${logFolder}\\ /p:HelixCorrelationInfoFileName=SubmittedHelixRuns.txt"

submittedHelixJson = readJSON file: "${logFolder}\\SubmittedHelixRuns.txt"
}
}
}
}

if (submitToHelix) {
stage ('Execute Tests') {
def contextBase
if (params.TestOuter) {
contextBase = "Win tests w/outer - ${params.AGroup} ${params.CGroup}"
}
else {
contextBase = "Win tests - ${params.AGroup} ${params.CGroup}"
}
waitForHelixRuns(submittedHelixJson, contextBase)
}
}
Loading

0 comments on commit e1fe61f

Please sign in to comment.