azure-pipelines.yml

# CI and PR triggers
trigger:
  branches:
    include:
    - main
    - feature/*
    - hotfix/*
    - release/*
  paths:
    include:
    - '*'
    exclude:
    - '**/*.md'
    - .config/*
    - .devcontainer/*
    - .github/*
    - .vscode/*
    - assets/*
    - docs/*
    - images/*
    - .editorconfig
    - .gitignore
    - '*.txt'
    - '*.github-issues'

pr:
  branches:
    include:
    - main
    - feature/*
    - hotfix/*
    - release/*
  paths:
    include:
    - '*'
    exclude:
    - '**/*.md'
    - .config/*
    - .devcontainer/*
    - .github/*
    - .vscode/*
    - assets/*
    - docs/*
    - images/*
    - .editorconfig
    - .gitignore
    - '*.txt'
    - '*.github-issues'

# Variables
variables:
  - name: _TeamName
    value: DotNetInteractive
  - name: _BuildConfig
    value: Release
  - name: _PublishUsingPipelines
    value: true
  - name: _DotNetArtifactsCategory
    value: .NETCore
  - name: DisableDockerDetector
    value: true
  - name: PocketLoggerLogPath
    value: $(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)
  - name: TryDotNetPackagesPath
    value: $(Build.SourcesDirectory)/artifacts/.trydotnet/packages
  - name: NodeJSVersion
    value: '16.13.0'
  - name: Codeql.Enabled
    value: true
  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
    - name: _DotNetValidationArtifactsCategory
      value: .NETCoreValidation
    - group: DotNet-Interactive-SDLValidation-Params
  - template: /eng/common/templates/variables/pool-providers.yml

stages:
- stage: build
  displayName: Build and Test
  jobs:
  - template: /eng/common/templates/jobs/jobs.yml
    parameters:
      enableMicrobuild: true
      enablePublishBuildArtifacts: true
      enablePublishTestResults: true
      enablePublishBuildAssets: true
      enablePublishUsingPipelines: $(_PublishUsingPipelines)
      jobs:
      - job: Windows_NT
        pool:
          ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
            name: $(DncEngPublicBuildPool)
            demands: ImageOverride -equals windows.vs2022.amd64.open
          ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
            name: $(DncEngInternalBuildPool)
            demands: ImageOverride -equals windows.vs2022.amd64
        timeoutInMinutes: 90
        variables:
        # Enable signing for internal, non-PR builds
        - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
          - group: DotNet-Blob-Feed
          - group: DotNet-Symbol-Server-Pats
          - name: _SignType
            value: Real
          - name: _DotNetPublishToBlobFeed
            value: true
          - name: _BuildArgs
            value: /p:SignType=$(_SignType)
                   /p:DotNetSignType=$(_SignType)
                   /p:MicroBuild_SigningEnabled=true
                   /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
                   /p:TeamName=$(_TeamName)
                   /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
                   /p:DotNetPublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
                   /p:DotNetPublishToBlobFeed=true
                   /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
                   /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
                   /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
                   /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
                   /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
                   /p:PublishToSymbolServer=true
        # else
        - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
          - name: _SignType
            value: Test
          - name: _BuildArgs
            value: /p:SignType=$(_SignType)
        steps:
        - checkout: self
          clean: true

        # Azure DevOps doesn't support git symlinks on Windows so we have to fake it
        - pwsh: . "$(Build.SourcesDirectory)\src\ensure-symlinks.ps1"
          displayName: ensure symlinks

        - task: NodeTool@0
          displayName: Add NodeJS/npm
          inputs:
            versionSpec: $(NodeJSVersion)

        - task: UseDotNet@2
          displayName: Use .NET SDK
          inputs:
            packageType: sdk
            useGlobalJson: true
            includePreviewVersions: true
            workingDirectory: $(Build.SourcesDirectory)
            installationPath: $(Agent.ToolsDirectory)/dotnet

        - script: |
            robocopy "eng\resources" "$(Build.SourcesDirectory)\artifacts"
            :: robocopy return codes are terrible; 1 means files were copied
            if "%errorlevel%" == "1" exit /b 0
            exit /b 1
          displayName: Prevent test directory crawling

        - pwsh: |
            $testArg = if ($env:SKIPTESTS -ne "true") { "-test" } else { "" }
            Write-Host "##vso[task.setvariable variable=_TestArgs]$testArg"
          displayName: Promote variables

        - script: eng\CIBuild.cmd
            -configuration $(_BuildConfig)
            -prepareMachine
            -sign
            $(_BuildArgs)
            $(_TestArgs)
          displayName: Build
          env:
            POCKETLOGGER_LOG_PATH: $(PocketLoggerLogPath)
            TRYDOTNET_PACKAGES_PATH: $(TryDotNetPackagesPath)
            DOTNET_INTERACTIVE_FRONTEND_NAME: CI

        - pwsh: ./test-retry-runner.ps1 -buildConfig $env:BUILDCONFIG
          displayName: Test / Blame
          workingDirectory: $(Build.SourcesDirectory)
          env:
            BUILDCONFIG: $(_BuildConfig)
          condition: ne(variables['SkipTests'], 'true')

        - pwsh: Get-ChildItem *.dmp -Recurse | Remove-Item
          displayName: Delete dump files
          workingDirectory: $(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)
          condition: and(ne(variables['KeepDumps'], 'true'), ne(variables['SkipTests'], 'true'))

        - task: PublishBuildArtifacts@1
          displayName: Publish Test results and Blame dumps
          inputs:
            pathToPublish: $(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)
            artifactName: Windows_test_results_and_dumps
            artifactType: container
          condition: failed()

        # publish VS Code and npm test results
        - task: PublishTestResults@2
          displayName: Publish VS Code extension and npm test results
          inputs:
            testResultsFormat: VSTest
            testResultsFiles: '**/*.trx'
            searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults'
          condition: always()

        # pack and publish vscode and npm
        - task: PowerShell@2
          displayName: Pack VS Code Extensions
          inputs:
            filePath: $(Build.SourcesDirectory)/eng/package/PackVSCodeExtension.ps1
            arguments: -stableToolVersionNumber $(StableToolVersionNumber) -outDir "$(Build.ArtifactStagingDirectory)\vscode"
            workingDirectory: "$(Build.SourcesDirectory)/src"
            pwsh: true

        - task: PublishBuildArtifacts@1
          displayName: Publish VSCode extension artifacts
          inputs:
            pathToPublish: $(Build.ArtifactStagingDirectory)\vscode
            artifactName: vscode
            artifactType: container

        - task: PowerShell@2
          displayName: Pack NPM package
          inputs:
            filePath: $(Build.SourcesDirectory)/eng/package/PackNpmPackage.ps1
            arguments: -packageVersionNumber $(StableToolVersionNumber) -outDir "$(Build.ArtifactStagingDirectory)\npm"
            workingDirectory: "$(Build.SourcesDirectory)/src/polyglot-notebooks"
            pwsh: true

        - task: PublishBuildArtifacts@1
          displayName: Publish NPM package artifacts
          inputs:
            pathToPublish: $(Build.ArtifactStagingDirectory)\npm
            artifactName: npm
            artifactType: container

        # Prevent symbols packages from being saved in the following `packages` artifact because they're incomplete.
        # See `eng/AfterSolutionBuild.targets:StripFilesFromSymbolPackages` for details.
        - script: del /S $(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)\*.symbols.nupkg
          displayName: Clean symbol packages

        - task: PublishBuildArtifacts@1
          displayName: Publish packages to artifacts container
          inputs:
            pathToPublish: $(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)
            artifactName: packages
            artifactType: container

  - template: /eng/common/templates/jobs/jobs.yml
    parameters:
      enableMicrobuild: true
      enablePublishBuildArtifacts: true
      enablePublishTestResults: true
      enablePublishBuildAssets: false
      enablePublishUsingPipelines: false
      jobs:
      - job: Linux
        pool:
          ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
            name: $(DncEngPublicBuildPool)
            demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
          ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
            name: $(DncEngInternalBuildPool)
            demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
        variables:
        # Enable signing for internal, non-PR builds
        - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
          - name: _SignType
            value: Test
          - name: _BuildArgs
            value: /p:SignType=$(_SignType)
              /p:DotNetSignType=$(_SignType)
              /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
              /p:TeamName=$(_TeamName)
              /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
        # else
        - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
          - name: _SignType
            value: Test
          - name: _BuildArgs
            value: /p:SignType=$(_SignType)
        steps:
        - script: git config --global core.longpaths true
          displayName: Enable `git clean` to handle long paths

        - checkout: self
          clean: true

        - task: NodeTool@0
          displayName: Add NodeJS/npm
          inputs:
            versionSpec: $(NodeJSVersion)

        - task: UseDotNet@2
          displayName: Use .NET SDK
          inputs:
            packageType: sdk
            useGlobalJson: true
            includePreviewVersions: true
            workingDirectory: $(Build.SourcesDirectory)
            installationPath: $(Agent.ToolsDirectory)/dotnet

        - script: |
            mkdir -p "$(Build.SourcesDirectory)/artifacts"
            rm -rf "$(Build.SourcesDirectory)/artifacts/*"
            cp eng/resources/* "$(Build.SourcesDirectory)/artifacts"
          displayName: Prevent test directory crawling

        - pwsh: |
            $testArg = if ($env:SKIPTESTS -ne "true") { "--test" } else { "" }
            Write-Host "##vso[task.setvariable variable=_TestArgs]$testArg"
          displayName: Promote variables

        - script: ./eng/cibuild.sh
            --configuration $(_BuildConfig)
            --prepareMachine
            $(_TestArgs)
          displayName: Build
          env:
            POCKETLOGGER_LOG_PATH: $(PocketLoggerLogPath)
            TRYDOTNET_PACKAGES_PATH: $(TryDotNetPackagesPath)
            DOTNET_INTERACTIVE_FRONTEND_NAME: CI

        - pwsh: ./test-retry-runner.ps1 -buildConfig $env:BUILDCONFIG
          displayName: Test / Blame
          workingDirectory: $(Build.SourcesDirectory)
          env:
            BUILDCONFIG: $(_BuildConfig)
          condition: ne(variables['SkipTests'], 'true')

        - pwsh: Get-ChildItem *.dmp -Recurse | Remove-Item
          displayName: Delete dump files
          workingDirectory: $(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)
          condition: and(ne(variables['KeepDumps'], 'true'), ne(variables['SkipTests'], 'true'))

        - task: PublishBuildArtifacts@1
          displayName: Publish Test results and Blame dumps
          inputs:
            pathToPublish: $(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)
            artifactName: Linux_test_results_and_dumps
            artifactType: container
          condition: failed()

        # publish VS Code and npm test results
        - task: PublishTestResults@2
          displayName: Publish VS Code extension and npm test results
          inputs:
            testResultsFormat: VSTest
            testResultsFiles: '**/*.trx'
            searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults'
          condition: always()

#---------------------------------------------------------------------------------------------------------------------#
#                                                    Post Build                                                       #
#---------------------------------------------------------------------------------------------------------------------#
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
  - template: eng/common/templates/post-build/post-build.yml
    parameters:
      publishingInfraVersion: 3
      # signing validation currently has issues with dotnet 7; disabling as per internal guidance
      enableSigningValidation: false
      # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved.
      enableSymbolValidation: false
      # SourceLink improperly looks for generated files.  See https://github.com/dotnet/arcade/issues/3069
      enableSourceLinkValidation: false
      # Enable SDL validation, passing through values from the 'DotNet-FSharp-SDLValidation-Params' group.
      SDLValidationParameters:
        enable: true
        params: >-
          -SourceToolsList @("policheck","credscan")
          -TsaInstanceURL $(_TsaInstanceURL)
          -TsaProjectName $(_TsaProjectName)
          -TsaNotificationEmail $(_TsaNotificationEmail)
          -TsaCodebaseAdmin $(_TsaCodebaseAdmin)
          -TsaBugAreaPath $(_TsaBugAreaPath)
          -TsaIterationPath $(_TsaIterationPath)
          -TsaRepositoryName "Interactive"
          -TsaCodebaseName "Interactive-GitHub"
          -TsaPublish $True
          -PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)/eng/policheck_exclusions.xml")

#---------------------------------------------------------------------------------------------------------------------#
#                                                    NPM Publish                                                      #
#---------------------------------------------------------------------------------------------------------------------#
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
  - template: eng/publish/publish-npm.yml
    parameters:
      packageName: microsoft-polyglot-notebooks-*.tgz
      registryUrl: pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/npm/registry/
      registryUser: dnceng
      registryEmail: dnceng@microsoft.com
      publishingBranch: main