diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 975102560f..8f4649cf5b 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -6,7 +6,7 @@ This file should be imported by eng/Versions.props - 11.0.0-beta.26409.102 + 12.0.0-beta.26461.114 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index df98dadfa1..5d4ada7804 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,12 +1,12 @@ - + - + https://github.com/dotnet/dotnet - 7fb8cef14d9ae6bd729b04638f88d00f1ba7eb99 + 7cfa7a8277e6c63b36482018162a9c2dd8922a96 diff --git a/eng/Versions.props b/eng/Versions.props index 520c87373e..bae9a1a036 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -3,8 +3,8 @@ - 3.0.0 - rc + 12.0.0 + alpha 1 diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1 index 6b5899d7a2..ec005e487c 100644 --- a/eng/common/Get-GitHubAppToken.ps1 +++ b/eng/common/Get-GitHubAppToken.ps1 @@ -1,13 +1,11 @@ # Mints a short-lived GitHub App installation access token by signing a JWT -# with a private key stored in Azure Key Vault (RSA, RS256). The signed JWT is -# exchanged with the GitHub API for a token scoped to a single installation. +# with an RSA private key (RS256). The signed JWT is exchanged with the GitHub +# API for a token scoped to a single installation. # # Requirements: -# - A GitHub App whose private key has been uploaded into Key Vault as an RSA -# key (the PEM converted to a Key Vault *key*, NOT stored as a secret). -# - The caller (the federated Azure service connection used to run this script) -# must have the `Key Vault Crypto User` role (or at minimum the `Sign` -# action) on that key. +# - A GitHub App ID and PEM private key stored as Azure Key Vault secrets. +# - The federated Azure service connection running this script must have +# `Get` access to those two secrets. # - The App must be installed on the target organization/account # (`InstallationOwner`) with the permissions/repositories it needs. # @@ -16,17 +14,17 @@ [CmdletBinding()] param( - # Name of the Key Vault that holds the GitHub App's RSA signing key. + # Name of the Key Vault holding the GitHub App credentials. [Parameter(Mandatory = $true)] [string] $KeyVaultName, - # Name of the RSA key inside the Key Vault (the App's private key). + # Secret Manager projection containing the GitHub App ID. [Parameter(Mandatory = $true)] - [string] $KeyName, + [string] $AppIdSecretName, - # The GitHub App's Client ID (the value to put in the `iss` JWT claim). + # Secret Manager projection containing the PEM private key. [Parameter(Mandatory = $true)] - [string] $AppClientId, + [string] $AppPrivateKeySecretName, # Login of the organization or user account whose installation we should # mint the token for (e.g. `dotnet`, `microsoft`). @@ -39,16 +37,69 @@ param( [Parameter(Mandatory = $false)] [string] $OutputVariableName ) - $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true . $PSScriptRoot\pipeline-logging-functions.ps1 +if ($KeyVaultName -notmatch '^[A-Za-z][A-Za-z0-9-]{1,22}[A-Za-z0-9]$' -or $KeyVaultName.Contains('--')) { + Write-PipelineTelemetryError -Category 'Build' -Message "KeyVaultName '$KeyVaultName' is not a valid Azure Key Vault name." + exit 1 +} + function ConvertTo-Base64Url([byte[]] $bytes) { return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') } +$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference +try { + # Azure CLI can emit non-fatal Python warnings to stderr. + $PSNativeCommandUseErrorActionPreference = $false + $keyVaultAccessToken = az account get-access-token ` + --resource https://vault.azure.net ` + --query accessToken ` + --output tsv ` + --only-show-errors + $tokenExitCode = $LASTEXITCODE +} +catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to acquire an Azure Key Vault access token: $_" + exit 1 +} +finally { + $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference +} +if ($tokenExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($keyVaultAccessToken)) { + Write-PipelineTelemetryError -Category 'Build' -Message "'az account get-access-token' exited with code $tokenExitCode while acquiring an Azure Key Vault access token." + exit 1 +} + +function Get-KeyVaultSecret([string] $SecretName) { + # Use the data-plane REST API because `az keyvault secret show` can fail + # with Errno 22 on hosted Windows agents when reading these projections. + $escapedSecretName = [Uri]::EscapeDataString($SecretName) + $secretUri = "https://$KeyVaultName.vault.azure.net/secrets/$escapedSecretName`?api-version=7.4" + try { + $response = Invoke-RestMethod ` + -Uri $secretUri ` + -Headers @{ Authorization = "Bearer $keyVaultAccessToken" } ` + -Method Get + } + catch { + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to read secret '$SecretName' from vault '$KeyVaultName': $_. Verify the secret exists and the service connection has 'Key Vault Secrets User' access to it." + exit 1 + } + if ([string]::IsNullOrWhiteSpace($response.value)) { + Write-PipelineTelemetryError -Category 'Build' -Message "Secret '$SecretName' in vault '$KeyVaultName' is empty." + exit 1 + } + return [string] $response.value +} + +Write-Host "Reading GitHub App credentials from vault '$KeyVaultName'..." +$appId = Get-KeyVaultSecret $AppIdSecretName +$privateKey = Get-KeyVaultSecret $AppPrivateKeySecretName + # Build JWT header and payload. Use [ordered] hashtables so JSON # serialization is deterministic. $jwtHeader = [ordered]@{ @@ -59,46 +110,38 @@ $now = [System.DateTimeOffset]::UtcNow $jwtPayload = [ordered]@{ iat = $now.AddMinutes(-1).ToUnixTimeSeconds() exp = $now.AddMinutes(5).ToUnixTimeSeconds() - iss = $AppClientId + iss = $appId } $headerEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtHeader | ConvertTo-Json -Compress))) $payloadEncoded = ConvertTo-Base64Url ([System.Text.Encoding]::UTF8.GetBytes(($jwtPayload | ConvertTo-Json -Compress))) $signingInput = "$headerEncoded.$payloadEncoded" -# Key Vault `sign` expects the *digest* (base64), not the raw bytes. -$sha256 = [System.Security.Cryptography.SHA256]::Create() -$digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput)) -$digestBase64 = [Convert]::ToBase64String($digestBytes) +$sha256 = [System.Security.Cryptography.SHA256]::Create() +try { + $digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signingInput)) +} +finally { + $sha256.Dispose() +} -Write-Host "Signing JWT with key '$KeyName' in vault '$KeyVaultName'..." -$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference +Write-Host 'Signing JWT with the GitHub App private key...' +$rsa = [System.Security.Cryptography.RSA]::Create() try { - # Azure CLI can emit non-fatal Python warnings to stderr even when signing succeeds. - # Use the exit code to determine success for this invocation. - $PSNativeCommandUseErrorActionPreference = $false - $signatureBase64 = az keyvault key sign ` - --vault-name $KeyVaultName ` - --name $KeyName ` - --algorithm RS256 ` - --digest $digestBase64 ` - --query signature ` - --output tsv ` - --only-show-errors - $signExitCode = $LASTEXITCODE + $rsa.ImportFromPem($privateKey) + $signatureBytes = $rsa.SignHash( + $digestBytes, + [System.Security.Cryptography.HashAlgorithmName]::SHA256, + [System.Security.Cryptography.RSASignaturePadding]::Pkcs1) + $signatureUrl = ConvertTo-Base64Url $signatureBytes } catch { - Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the JWT via Key Vault (key '$KeyName', vault '$KeyVaultName'): $_. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the GitHub App JWT with the supplied private key: $_" exit 1 } finally { - $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference + $rsa.Dispose() } -if ($signExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($signatureBase64)) { - Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $signExitCode for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." - exit 1 -} -$signatureUrl = $signatureBase64.Trim().TrimEnd('=').Replace('+', '-').Replace('/', '_') $jwt = "$signingInput.$signatureUrl" $headers = @{ @@ -110,27 +153,38 @@ $headers = @{ Write-Host "Looking up installation for '$InstallationOwner'..." try { - $installations = @() + $installations = [System.Collections.Generic.List[object]]::new() $page = 1 do { - $pageInstallations = @(Invoke-RestMethod ` + $pageResponse = Invoke-RestMethod ` -Uri "https://api.github.com/app/installations?per_page=100&page=$page" ` -Headers $headers ` - -Method Get) - $installations += $pageInstallations + -Method Get + $pageInstallationCount = 0 + foreach ($installation in $pageResponse) { + $installations.Add($installation) + $pageInstallationCount++ + } $page++ - } while ($pageInstallations.Count -eq 100) + } while ($pageInstallationCount -eq 100) } catch { - Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect." + Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App ID may be incorrect." exit 1 } -$installation = $installations | Where-Object { $_.account.login -ieq $InstallationOwner } | Select-Object -First 1 -if (-not $installation) { +$matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner }) +if ($matchingInstallations.Count -eq 0) { $found = ($installations | ForEach-Object { $_.account.login }) -join ', ' Write-PipelineTelemetryError -Category 'Build' -Message "No installation found for '$InstallationOwner'. App is installed on: $found" exit 1 } +if ($matchingInstallations.Count -ne 1) { + $matchingIds = ($matchingInstallations | ForEach-Object { $_.id }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "Found multiple installations for '$InstallationOwner': $matchingIds" + exit 1 +} +$installation = $matchingInstallations[0] +Write-Host "Using installation $($installation.id) for '$($installation.account.login)'." try { $tokenResponse = Invoke-RestMethod ` diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index b3bddff355..9efd17273a 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -11,7 +11,7 @@ # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: # filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token +# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config # env: # Token: $(InternalFeedToken) # @@ -29,12 +29,14 @@ [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - $Password + # Keep the legacy name as an alias while callers migrate secrets to the Token environment variable. + [Alias("Password")]$Credential ) $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$feedCredential = if ($env:Token) { $env:Token } else { $Credential } # This script only consumes helper functions from tools.ps1 to configure NuGet feeds. # Skip importing configure-toolset.ps1 so that repo-specific toolset setup (e.g. acquiring @@ -44,14 +46,14 @@ $disableConfigureToolsetImport = $true . $PSScriptRoot\tools.ps1 # Adds or enables the package source with the given name -function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { - if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) { - AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password +function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $credential) { + if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName -Credential $credential)) { + AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $Username -credential $credential } } # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $credential) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -67,13 +69,13 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern Write-Host "Package source $SourceName already present and enabled." } - AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd + AddCredential -Creds $creds -Source $SourceName -Username $Username -credential $credential } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $pwd) { +function AddCredential($creds, $source, $username, $credential) { # If no cred supplied, don't do anything. - if (!$pwd) { + if (!$credential) { return; } @@ -108,19 +110,19 @@ function AddCredential($creds, $source, $username, $pwd) { $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $pwd) + $passwordElement.SetAttribute("value", $credential) } # Enable all darc-int package sources. -function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) { +function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds, $Credential) { $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") ForEach ($DisabledPackageSource in $maestroInternalSources) { - EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key + EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key -Credential $Credential } } # Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise. -function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) { +function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName, $Credential) { $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']") if ($DisabledPackageSource) { Write-Host "Enabling internal source '$($DisabledPackageSource.key)'." @@ -128,7 +130,7 @@ function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSo # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) - AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password + AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -credential $credential return $true } return $false @@ -153,7 +155,7 @@ if ($sources -eq $null) { $creds = $null $feedSuffix = "v3/index.json" -if ($Password) { +if ($feedCredential) { $feedSuffix = "v2" # Looks for a node. Create it if none is found. $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") @@ -169,16 +171,16 @@ $userName = "dn-bot" $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") if ($disabledSources -ne $null) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds + EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds -Credential $feedCredential } -$dotnetVersions = @('5','6','7','8','9','10') +$dotnetVersions = @('5','6','7','8','9','10','11') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password - AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -credential $feedCredential + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -credential $feedCredential } } diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 67e7e0942c..d4c66a98e1 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -24,7 +24,9 @@ # This logic is also abstracted into enable-internal-sources.yml. ConfigFile=$1 -CredToken=$2 +# Prefer the environment variable so credentials do not appear in process arguments. +# Retain the positional argument as a compatibility fallback for existing callers. +CredToken=${Token:-$2} NL='\n' TB=' ' @@ -167,7 +169,7 @@ if [ "$?" == "0" ]; then EnableMaestroInternalPackageSources fi -DotNetVersions=('5' '6' '7' '8' '9' '10') +DotNetVersions=('5' '6' '7' '8' '9' '10' '11') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index dd84699f50..fee2f83991 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -8,6 +8,7 @@ Param( [bool] $warnAsError = $true, [string] $warnNotAsError = '', [bool] $nodeReuse = $true, + [bool][Alias('mt')]$msbuildMultiThreaded = $false, [switch] $buildCheck = $false, [switch][Alias('r')]$restore, [switch] $deployDeps, @@ -79,6 +80,7 @@ function Print-Usage() { Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + Write-Host " -msbuildMultiThreaded Sets MSBuild's multi-threaded mode, i.e. the -mt switch ('1' or '0') (short: -mt)" Write-Host " -buildCheck Sets /check msbuild parameter" Write-Host " -fromVMR Set when building from within the VMR" Write-Host " -disablePipelineSetResult Set to disable masking the actual exit code in the pipeline when the build fails" @@ -175,9 +177,8 @@ try { if (-not $excludeCIBinarylog) { $binaryLog = $true } - # Disable node reuse on CI unless explicitly opted in via MSBUILD_NODEREUSE_ENABLED. - # Internal testing only; this env var will be replaced with a switch (https://github.com/dotnet/arcade/issues/17013) and must not be depended on. - if ($env:MSBUILD_NODEREUSE_ENABLED -ne "1") { + # Node reuse isn't used on CI unless it was explicitly requested via -nodeReuse. + if (-not $PSBoundParameters.ContainsKey('nodeReuse')) { $nodeReuse = $false } } diff --git a/eng/common/build.sh b/eng/common/build.sh index e37edd6cff..f65b048aa8 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -43,6 +43,7 @@ usage() echo " --pipelinesLog Promote msbuild errors/warnings to Azure Pipelines timeline issues; defaults to on in CI (short: -pl)" echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + echo " --msbuildMultiThreaded Sets MSBuild's multi-threaded mode, i.e. the -mt switch ('true' or 'false') (short: --mt)" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" echo " --warnNotAsError Sets a semi-colon delimited list of warning codes that should not be treated as errors" echo " --buildCheck Sets /check msbuild parameter" @@ -84,7 +85,9 @@ clean=false warn_as_error=true warn_not_as_error='' -node_reuse=true +# Empty means "not specified"; tools.sh defaults these to on for local builds and off on CI. +node_reuse='' +msbuild_multi_threaded='' build_check=false binary_log=false binary_log_name='' @@ -199,6 +202,10 @@ while [[ $# -gt 0 ]]; do node_reuse=$2 shift ;; + -msbuildmultithreaded|-mt) + msbuild_multi_threaded=$2 + shift + ;; -buildcheck) build_check=true ;; @@ -224,11 +231,6 @@ fi if [[ "$ci" == true ]]; then pipelines_log=true - # Disable node reuse on CI unless explicitly opted in via MSBUILD_NODEREUSE_ENABLED. - # Internal testing only; this env var will be replaced with a switch (https://github.com/dotnet/arcade/issues/17013) and must not be depended on. - if [[ "${MSBUILD_NODEREUSE_ENABLED:-}" != "1" ]]; then - node_reuse=false - fi if [[ "$exclude_ci_binary_log" == false ]]; then binary_log=true fi @@ -252,7 +254,7 @@ function Build { properties+=("/p:Projects=$projects") fi - local bl="" + local bl=() if [[ "$binary_log" == true ]]; then local binary_log_path="" if [[ -z "$binary_log_name" ]]; then @@ -264,7 +266,7 @@ function Build { fi mkdir -p "$(dirname "$binary_log_path")" - bl="/bl:\"$binary_log_path\"" + bl=("/bl:$binary_log_path") fi local check="" @@ -272,8 +274,8 @@ function Build { check="/check" fi - MSBuild $_InitializeToolset \ - $bl \ + MSBuild "$_InitializeToolset" \ + ${bl[@]+"${bl[@]}"} \ $check \ /p:Configuration=$configuration \ /p:RepoRoot="$repo_root" \ @@ -297,7 +299,7 @@ function Build { if [[ "$clean" == true ]]; then if [ -d "$artifacts_dir" ]; then - rm -rf $artifacts_dir + rm -rf "$artifacts_dir" echo "Artifacts directory deleted." fi exit 0 diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index a65b50d0a7..53bbf74927 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -62,6 +62,12 @@ parameters: type: number default: 30 +# Maximum number of work items whose results may be downloaded, parsed, and +# uploaded concurrently. +- name: testResultUploadParallelism + type: number + default: 48 + # When 'true' (the default), Helix work items that exit 0 but have failed AzDO test results # are treated as failed: they count toward the monitor's exit code and are resubmitted by a # later invocation's retry pass. Set to 'false' to fall back to exit-code-only outcomes. @@ -84,6 +90,15 @@ parameters: type: boolean default: false +# Controls per-test output attachments. Defaults to Failed. +- name: testResultAttachmentMode + type: string + default: Failed + values: + - Failed + - All + - None + # Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool # nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into # a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is @@ -206,10 +221,13 @@ jobs: --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. --stage-name '$(System.StageName)' --stage-attempt '$(System.StageAttempt)' + --job-attempt '$(System.JobAttempt)' + --test-result-upload-parallelism '${{ parameters.testResultUploadParallelism }}' ) organization='${{ parameters.organization }}' repository='${{ parameters.repository }}' + testResultAttachmentMode='${{ parameters.testResultAttachmentMode }}' # Fall back to Azure DevOps-provided environment variables when the caller did not # supply organization / repository explicitly. BUILD_REPOSITORY_NAME is typically @@ -232,6 +250,9 @@ jobs: if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi + if [ -n "$testResultAttachmentMode" ]; then + toolArgs+=( --test-result-attachment-mode "$testResultAttachmentMode" ) + fi # Build.Reason and Build.SourceBranch are required to derive the Helix source filter # the same way the Helix SDK submitter does (PR -> 'pr', internal -> 'official', diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index cb60f52978..2716ecd18f 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -28,6 +28,7 @@ parameters: enablePublishTestResults: false enablePublishing: false enableBuildRetry: false + enableAstred: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' @@ -119,6 +120,12 @@ jobs: - name: ${{ pair.key }} value: ${{ pair.value }} + - ${{ if and(eq(parameters.enableAstred, true), eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: MSBUILDDEBUGENGINE + value: 1 + - name: MSBUILDDEBUGPATH + value: $(Build.ArtifactStagingDirectory)/AstredCapture/binlogs + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-HelixApi-Access @@ -236,3 +243,8 @@ jobs: condition: always() - ${{ each step in parameters.artifactPublishSteps }}: - ${{ step }} + + - ${{ if and(eq(parameters.enableAstred, true), eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/core-templates/steps/astred-artifacts.yml + parameters: + binlogDir: $(MSBUILDDEBUGPATH) diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index 4f5653d73a..b772dc5788 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -5,23 +5,14 @@ parameters: # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: '' - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). - # When set, dnceng/internal builds acquire a federated Entra token instead of using a PAT. - # All other projects (e.g. DevDiv, public), where this dnceng-scoped service connection does not - # exist, and any pipeline that sets this to '' fall back to PAT-based auth via the CeapexPat parameter. + # Project-scoped WIF service connection for Ceapex feed authentication. CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' - # GitHub App authentication for the OneLoc check-in PR (dnceng/internal only). - # The infrastructure identifiers are centralized here and the App path is enabled by default. - # DevDiv requires its own project-scoped service connection before this path can be enabled there. - UseGitHubAppAuthentication: true + # GitHub App authentication for the OneLoc check-in PR. GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' - GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' GitHubAppKeyVaultName: 'EngKeyVault' - GitHubAppKeyName: 'oneloc-localization-app-key' + GitHubAppIdSecretName: 'oneloc-localization-app-app-id' + GitHubAppPrivateKeySecretName: 'oneloc-localization-app-app-private-key' SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true @@ -49,7 +40,6 @@ jobs: displayName: OneLocBuild${{ parameters.JobNameSuffix }} variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - name: _GenerateLocProjectArguments value: -SourcesDirectory ${{ parameters.SourcesDirectory }} -LanguageSet "${{ parameters.LanguageSet }}" @@ -80,6 +70,10 @@ jobs: steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + - ${{ if notIn(variables['System.TeamProject'], 'internal', 'DevDiv') }}: + - 'OneLocBuild is supported only in dnceng/internal and DevDiv/DevDiv.': error + - ${{ if eq(parameters.CeapexServiceConnection, '') }}: + - 'CeapexServiceConnection must identify a WIF service connection.': error - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - task: Powershell@2 @@ -89,25 +83,25 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} - # Acquire an Entra token for ceapex feed access via WIF (dnceng/internal only). - # All other projects use PAT-based auth, since the ceapex service connection is scoped to dnceng/internal. - - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: - - template: /eng/common/templates/steps/get-federated-access-token.yml - parameters: - federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} - outputVariableName: 'CeapexEntraToken' - condition: ${{ parameters.condition }} + # Acquire a short-lived Entra token for Ceapex feed access. + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} + outputVariableName: 'CeapexEntraToken' + condition: ${{ parameters.condition }} - # Mint a short-lived GitHub App installation token for the loc check-in PR (dnceng/internal only). - # All other projects fall back to PAT-based auth, since the app service connection is scoped to dnceng/internal. - - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), eq(variables['System.TeamProject'], 'internal')) }}: + # Mint a short-lived GitHub App installation token for the loc check-in PR. + - ${{ if eq(parameters.RepoType, 'gitHub') }}: - template: /eng/common/core-templates/steps/get-github-app-token.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}: + azureSubscription: 'devdiv-oneloc-githubapp' + ${{ else }}: + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} - keyName: ${{ parameters.GitHubAppKeyName }} - appClientId: ${{ parameters.GitHubAppClientId }} + appIdSecretName: ${{ parameters.GitHubAppIdSecretName }} + appPrivateKeySecretName: ${{ parameters.GitHubAppPrivateKeySecretName }} installationOwner: ${{ parameters.GitHubOrg }} outputVariableName: 'GitHubAppInstallationToken' condition: ${{ parameters.condition }} @@ -127,16 +121,10 @@ jobs: isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: - patVariable: $(CeapexEntraToken) - ${{ if or(eq(parameters.CeapexServiceConnection, ''), ne(variables['System.TeamProject'], 'internal')) }}: - patVariable: ${{ parameters.CeapexPat }} + patVariable: $(CeapexEntraToken) ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), eq(variables['System.TeamProject'], 'internal')) }}: - gitHubPatVariable: "$(GitHubAppInstallationToken)" - ${{ if or(eq(parameters.UseGitHubAppAuthentication, false), ne(variables['System.TeamProject'], 'internal')) }}: - gitHubPatVariable: "${{ parameters.GithubPat }}" + gitHubPatVariable: "$(GitHubAppInstallationToken)" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index bac6ac5faa..b0dc8f1706 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -12,6 +12,7 @@ jobs: - job: SourceIndexStage1 dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} + continueOnError: true variables: - name: BinlogPath value: ${{ parameters.binlogPath }} @@ -38,9 +39,11 @@ jobs: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository + - ${{ if ne(parameters.sourceIndexBuildCommand, '') }}: + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml parameters: binLogPath: ${{ parameters.binLogPath }} + runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 9d95135269..6dcee6664d 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -236,6 +236,7 @@ stages: StageLabel: 'Validation' JobLabel: 'Signing' BinlogToolVersion: $(BinlogToolVersion) + enableInternalRuntimes: false # SourceLink validation has been removed — the underlying CLI tool # (targeting netcoreapp2.1) has not functioned for years. diff --git a/eng/common/core-templates/steps/astred-artifacts.yml b/eng/common/core-templates/steps/astred-artifacts.yml new file mode 100644 index 0000000000..b914082f58 --- /dev/null +++ b/eng/common/core-templates/steps/astred-artifacts.yml @@ -0,0 +1,103 @@ +# Astred footer for producing and uploading a portable digest. +# The calling job must configure its header before any build steps run: +# MSBUILDDEBUGENGINE=1 +# MSBUILDDEBUGPATH= +parameters: +- name: sourcesPath + type: string + default: $(Build.SourcesDirectory) +- name: binlogDir + type: string + default: $(Build.ArtifactStagingDirectory)/AstredCapture/binlogs +- name: capturePath + type: string + default: $(Build.ArtifactStagingDirectory)/AstredCapture + +steps: +- task: AstredInstaller@0 + displayName: Install Astred CLI + inputs: + Version: '2.14.1' + FeedUrl: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet-internal-FoSSE/nuget/v3/index.json' + +- pwsh: | + $ErrorActionPreference = 'Continue' + $apjOut = Join-Path $env:ASTRED_CAPTURE_PATH 'apj' + New-Item -ItemType Directory -Force -Path $apjOut | Out-Null + + $binlogs = Get-ChildItem -Path $env:ASTRED_BINLOG_DIR -Recurse -Force -Filter *.binlog ` + -ErrorAction SilentlyContinue + if (-not $binlogs) { + Write-Host "##vso[task.logissue type=warning]No binlogs found in $env:ASTRED_BINLOG_DIR. Check the calling job's Astred header configuration for MSBUILDDEBUGENGINE / MSBuildDebugEngine and MSBUILDDEBUGPATH." + exit 0 + } + + $project = Join-Path $apjOut '.astred.project.json' + $binlogPaths = @($binlogs.FullName) + Write-Host "astproj: processing $($binlogPaths.Count) binlog(s)" + astred astproj -nofolders @binlogPaths "-o:$project" + if ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=warning]astproj failed (exit $LASTEXITCODE)" + } + displayName: Generate Astred Project Files + workingDirectory: ${{ parameters.sourcesPath }} + env: + ASTRED_BINLOG_DIR: ${{ parameters.binlogDir }} + ASTRED_CAPTURE_PATH: ${{ parameters.capturePath }} + condition: succeededOrFailed() + continueOnError: true + +- pwsh: | + $ErrorActionPreference = 'Continue' + $apjDir = Join-Path $env:ASTRED_CAPTURE_PATH 'apj' + $uploadRoot = Join-Path $env:ASTRED_CAPTURE_PATH 'upload' + $project = Join-Path $apjDir '.astred.project.json' + + if (-not (Test-Path $project)) { + Write-Host "##vso[task.logissue type=warning]No Astred project file was produced." + exit 0 + } + + $digest = Join-Path $apjDir '.astred.digest.zip' + Remove-Item $digest -ErrorAction SilentlyContinue + astred "-repo:$env:ASTRED_SOURCES_PATH" "-project:$project" -digest + if ($LASTEXITCODE -eq 0 -and (Test-Path $digest)) { + $commitTimeText = & git -C $env:ASTRED_SOURCES_PATH show -s --format=%cI $env:BUILD_SOURCEVERSION + if ($LASTEXITCODE -ne 0) { + throw "Could not read the commit timestamp for $env:BUILD_SOURCEVERSION." + } + + $commitTime = [DateTimeOffset]::Parse( + $commitTimeText.Trim(), + [Globalization.CultureInfo]::InvariantCulture) + $eventFolder = '{0}_{1}' -f ` + $commitTime.UtcDateTime.ToString('yyyy-MM-ddTHH-mm-ssZ'), ` + $env:BUILD_SOURCEVERSION + $targetDir = Join-Path (Join-Path $uploadRoot 'AST') $eventFolder + $target = Join-Path $targetDir '.astred.digest.zip' + New-Item -ItemType Directory -Force -Path $targetDir | Out-Null + Move-Item $digest $target -Force + Write-Host "Prepared Astred digest: $target" + Write-Host "##vso[task.setvariable variable=ASTRED_DIGEST_READY]true" + } + elseif ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=warning]Digest generation failed for $project (exit $LASTEXITCODE)" + Remove-Item $digest -ErrorAction SilentlyContinue + } + else { + Write-Host "##vso[task.logissue type=warning]Digest not produced for $project" + } + displayName: Package Portable Astred Digests + env: + ASTRED_SOURCES_PATH: ${{ parameters.sourcesPath }} + ASTRED_CAPTURE_PATH: ${{ parameters.capturePath }} + condition: succeededOrFailed() + continueOnError: true + +- task: UploadAstred@0 + displayName: Upload Digest to Astred + condition: and(succeededOrFailed(), eq(variables['ASTRED_DIGEST_READY'], 'true')) + inputs: + SourcePath: ${{ parameters.capturePath }}/upload + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/common/core-templates/steps/enable-internal-sources.yml b/eng/common/core-templates/steps/enable-internal-sources.yml index 51af9a0170..843cdff782 100644 --- a/eng/common/core-templates/steps/enable-internal-sources.yml +++ b/eng/common/core-templates/steps/enable-internal-sources.yml @@ -19,7 +19,7 @@ steps: displayName: Setup Internal Feeds inputs: filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config env: Token: ${{ parameters.legacyCredential }} - task: Bash@3 @@ -28,7 +28,7 @@ steps: inputs: targetType: inline script: | - "$(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh" "$(System.DefaultWorkingDirectory)/NuGet.config" "$Token" + "$(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh" "$(System.DefaultWorkingDirectory)/NuGet.config" env: Token: ${{ parameters.legacyCredential }} # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. @@ -58,13 +58,17 @@ steps: displayName: Setup Internal Feeds inputs: filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config + env: + Token: $(dnceng-artifacts-feeds-read-access-token) - task: Bash@3 condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) displayName: Setup Internal Feeds inputs: filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh - arguments: $(System.DefaultWorkingDirectory)/NuGet.config $(dnceng-artifacts-feeds-read-access-token) + arguments: $(System.DefaultWorkingDirectory)/NuGet.config + env: + Token: $(dnceng-artifacts-feeds-read-access-token) # This is required in certain scenarios to install the ADO credential provider. # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others # (e.g. dotnet msbuild). diff --git a/eng/common/core-templates/steps/get-github-app-token.yml b/eng/common/core-templates/steps/get-github-app-token.yml index 6d42a48d3c..3eeb5a4c1b 100644 --- a/eng/common/core-templates/steps/get-github-app-token.yml +++ b/eng/common/core-templates/steps/get-github-app-token.yml @@ -1,13 +1,11 @@ # Mints a short-lived GitHub App installation access token by signing a JWT -# with a private key stored in Azure Key Vault (RSA, RS256). The JWT is -# exchanged with the GitHub API for a token scoped to a single installation. +# with an RSA private key (RS256). The JWT is exchanged with the GitHub API +# for a token scoped to a single installation. # # Requirements (per GitHub App you want to authenticate as): -# - A GitHub App with its private key uploaded into Key Vault as an RSA key -# (PEM converted to a key, NOT stored as a secret). -# - The Azure service connection passed via `azureSubscription` must be -# granted the `Key Vault Crypto User` role (or at minimum `Sign` action) -# on that key. +# - A GitHub App ID and PEM private key stored as Azure Key Vault secrets. +# - The Azure service connection passed via `azureSubscription` must have +# `Get` access to those two secrets. # - The App must be installed on the target organization/account # (`installationOwner`) with the permissions/repositories you need. # @@ -17,23 +15,18 @@ # enterprise classic-PAT lifetime policy. parameters: -# Azure DevOps service connection (federated) that can call -# `az keyvault key sign` on the App's signing key. +# Azure DevOps service connection (federated) that can read the App credentials. - name: azureSubscription type: string -# Name of the Key Vault that holds the GitHub App's RSA signing key. +# Name of the Key Vault holding Secret Manager's github-app-secret projections. - name: keyVaultName type: string -# Name of the RSA key inside the Key Vault (the App's private key). -- name: keyName +- name: appIdSecretName type: string -# The GitHub App's Client ID (the value to put in the `iss` JWT claim). -# Prefer this over the numeric App ID; GitHub accepts either, but Client ID -# is the documented form going forward. -- name: appClientId +- name: appPrivateKeySecretName type: string # Login of the organization or user account whose installation we should @@ -73,7 +66,7 @@ steps: inlineScript: | & "$(System.DefaultWorkingDirectory)/eng/common/Get-GitHubAppToken.ps1" ` -KeyVaultName '${{ parameters.keyVaultName }}' ` - -KeyName '${{ parameters.keyName }}' ` - -AppClientId '${{ parameters.appClientId }}' ` + -AppIdSecretName '${{ parameters.appIdSecretName }}' ` + -AppPrivateKeySecretName '${{ parameters.appPrivateKeySecretName }}' ` -InstallationOwner '${{ parameters.installationOwner }}' ` -OutputVariableName '${{ parameters.outputVariableName }}' diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 2c1e0ab116..244fef0890 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -5,6 +5,7 @@ parameters: # A default - in case value from eng/common/core-templates/post-build/common-variables.yml is not passed BinlogToolVersion: '1.0.11' is1ESPipeline: false + enableInternalRuntimes: true steps: - task: Powershell@2 @@ -25,13 +26,20 @@ steps: # Sensitive data can as well be added to $(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' # If the file exists - sensitive data for redaction will be sourced from it # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' - -BinlogToolVersion '${{parameters.BinlogToolVersion}}' - -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' - -runtimeSourceFeed https://ci.dot.net/internal - -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' - '$(System.AccessToken)' - ${{parameters.CustomSensitiveDataList}} + ${{ if and(eq(parameters.enableInternalRuntimes, true), ne(variables['System.TeamProject'], 'public')) }}: + arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' + -BinlogToolVersion '${{parameters.BinlogToolVersion}}' + -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' + '$(System.AccessToken)' + ${{parameters.CustomSensitiveDataList}} + ${{ else }}: + arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' + -BinlogToolVersion '${{parameters.BinlogToolVersion}}' + -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(System.AccessToken)' + ${{parameters.CustomSensitiveDataList}} continueOnError: true condition: always() diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index fdca622357..6d4173aaf7 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,7 +1,9 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20260521.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20260521.2 + runAsPublic: false + sourceIndexUploadPackageVersion: '2.0.0-20260521.2' + sourceIndexComplogPackageVersion: '*' sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexPublicPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog steps: @@ -14,14 +16,17 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: "Source Index: Download netsourceindex Tools" + $(Agent.TempDirectory)/dotnet/dotnet tool install complog --version "${{parameters.sourceIndexComplogPackageVersion}}" --source ${{parameters.sourceIndexPublicPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version "${{parameters.sourceIndexUploadPackageVersion}}" --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: "Source Index: Download Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: "Source Index: Process Binlog into indexable sln" +- script: | + mkdir ".source-index/stage1output" + git rev-parse HEAD > .source-index/stage1output/hash + $(Agent.TempDirectory)/.source-index/tools/complog create ${{parameters.BinlogPath}} -o .source-index/stage1output/build.complog + displayName: "Source Index: Process Binlog into Complog" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: AzureCLI@2 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index f58abbd2d1..3fea306bc2 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -532,27 +532,33 @@ ensureDownloadTool() } if [[ "$__CodeName" == "alpine" ]]; then - __ApkToolsVersion=2.12.11 + __ApkToolsVersion=2.14.4-r1 __ApkToolsDir="$(mktemp -d)" __ApkKeysDir="$(mktemp -d)" arch="$(uname -m)" __AlpineRepo="${__AlpineRepoOverride:-https://dl-cdn.alpinelinux.org/alpine}" ensureDownloadTool + __ApkToolsPackage="$__ApkToolsDir/apk-tools-static.apk" + __ApkToolsUrl="$__AlpineRepo/v3.20/main/$arch/apk-tools-static-$__ApkToolsVersion.apk" if [[ "$__hasWget" == 1 ]]; then - wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + wget -O "$__ApkToolsPackage" "$__ApkToolsUrl" else - curl -SLO --create-dirs --output-dir "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" + curl -fSL -o "$__ApkToolsPackage" "$__ApkToolsUrl" fi + if [[ "$arch" == "x86_64" ]]; then - __ApkToolsSHA512SUM="53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33" + __ApkToolsSHA512SUM="b1b3cc382aa0ec26a2c24b742701a1f9885d0678365f9aea15d3d005926b06ecc802659cec8a7deba2717af99c19c708a17c23e1f0f07742268ee5be5400eb9e" elif [[ "$arch" == "aarch64" ]]; then - __ApkToolsSHA512SUM="9e2b37ecb2b56c05dad23d379be84fd494c14bd730b620d0d576bda760588e1f2f59a7fcb2f2080577e0085f23a0ca8eadd993b4e61c2ab29549fdb71969afd0" + __ApkToolsSHA512SUM="61f9a636c5ac4e96e7a3f69fd65e60fc57b3ec8b23619c4df86f59b89e71d1309b3e406388945bdf0dd9168dac22df376943a70ff3efa179e5687e586f825fb0" else - echo "WARNING: add missing hash for your host architecture. To find the value, use: 'find /tmp -name apk.static -exec sha512sum {} \;'" + >&2 echo "ERROR: Unsupported apk-tools-static host architecture '$arch'." + exit 1 fi - echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c + echo "$__ApkToolsSHA512SUM $__ApkToolsPackage" | sha512sum -c + tar -xzf "$__ApkToolsPackage" -C "$__ApkToolsDir" --strip-components=1 sbin/apk.static + rm "$__ApkToolsPackage" chmod +x "$__ApkToolsDir/apk.static" if [[ "$__AlpineVersion" == "edge" ]]; then diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 3e6a8d6acf..4de1a04fee 100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -60,8 +60,8 @@ while (($# > 0)); do echo " - (default) %USERPROFILE%/.netcoreeng/native" echo "" echo " --clean Switch specifying not to install anything, but cleanup native asset folders" - echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" - echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" + echo " --donotabortonfailure Switch specifying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifying whether to display warnings during native tools installation on failure" echo " --force Clean and then install tools" echo " --help Print help and exit" echo "" @@ -83,7 +83,7 @@ function ReadGlobalJsonNativeTools { # KEY="" VALUE="" # followed by a null byte. # - # bash: read line with null byte delimeter and push to array (for later `eval`uation). + # bash: read line with null byte delimiter and push to array (for later `eval`uation). while IFS= read -rd '' line; do native_assets+=("$line") diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss index 5d892d6193..c810350e58 100644 --- a/eng/common/loc/P22DotNetHtmlLocalization.lss +++ b/eng/common/loc/P22DotNetHtmlLocalization.lss @@ -16,7 +16,7 @@ -