Skip to content

Commit 8f8366d

Browse files
dagooddagood
committed
Merge 'release/2.1' into master
# Conflicts: # BuildToolsVersion.txt # dependencies.props # dependencies.targets # netci.groovy # repos/cli.proj # repos/msbuild.proj # smoke-test.sh # src/cli # src/core-setup # src/coreclr # src/corefx
2 parents 0b508e0 + 4ca4d03 commit 8f8366d

16 files changed

+435
-13
lines changed

‎.vsts-ci.yml

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
phases:
2+
- phase: source_build_ci
3+
variables:
4+
buildLoggingOptions: /clp:v=detailed /p:MinimalConsoleLogOutput=false
5+
buildConfiguration: Release
6+
buildOfflineTarball: false
7+
docker.runInRoot: docker run --rm -v $(rootDirectory):/root -w /root
8+
docker.runInSrc: docker run --rm -v $(Build.SourcesDirectory):/src -w /src
9+
docker.runInSourceBuild: docker run --rm -v $(rootDirectory)/sb/source-build:/src -w /src
10+
docker.runInTarball: docker run --rm -v $(rootDirectory)/sb/tarball/$(tarballName):/tb -w /tb
11+
dropDirectory: $(stagingDirectory)/drop
12+
rootDirectory: $(Build.SourcesDirectory)/..
13+
stagingDirectory: $(rootDirectory)/sb/staging
14+
tarballName: tarball_$(Build.BuildId)
15+
16+
queue:
17+
name: DotNet-Build
18+
demands: agent.os -equals linux
19+
timeoutInMinutes: 240
20+
parallel: 2
21+
matrix:
22+
centos71:
23+
imageName: microsoft/dotnet-buildtools-prereqs:centos711503_prereqs_2
24+
rhel7-unshared:
25+
imageName: microsoft/dotnet-buildtools-prereqs:rhel7_prereqs_2
26+
buildOfflineTarball: true
27+
28+
steps:
29+
- template: ./.vsts.pipelines/steps/docker-cleanup-linux.yml
30+
31+
# create working directory and copy source into it
32+
- script: |
33+
set -x
34+
$(docker.runInRoot) $(imageName) rm -rf /root/sb/
35+
$(docker.runInRoot) $(imageName) mkdir -p /root/sb/tarball
36+
$(docker.runInSrc) -v $(rootDirectory):/root $(imageName) cp -r . /root/sb/source-build
37+
displayName: Clean sb directory and copy source from cloned directory
38+
39+
# fetch vsts commits if building internally
40+
- script: |
41+
set -x
42+
# Ignore failure for the first command. It will intentionally fail if the commit is only
43+
# available in VSTS. "submodule update --init" is the simplest way to set up the submodule
44+
# directory. ("submodule init" only sets up .git/config, not the e.g. src/coreclr/.git and
45+
# .git/modules/src/coreclr/ directories.)
46+
$(docker.runInSourceBuild) $(imageName) git submodule update --init --recursive
47+
$(docker.runInSourceBuild) $(imageName) ./fetch-vsts-commits.sh $(user.PAT)
48+
displayName: Fetch internal vsts commits
49+
condition: and(succeeded(), ne(variables['user.PAT'], ''))
50+
51+
# init submodules
52+
- script: $(docker.runInSourceBuild) $(imageName) git submodule update --init --recursive
53+
displayName: Initialize submodules
54+
55+
# build source-build
56+
- script: $(docker.runInSourceBuild) $(imageName) ./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=$(buildConfiguration) /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix) $(buildLoggingOptions)
57+
displayName: Build source-build
58+
timeoutInMinutes: 90
59+
60+
# copy logs to working directory
61+
- script: |
62+
set -x
63+
$(docker.runInSourceBuild) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/source-build/binlogs; find . -name '*.binlog' -exec cp {} /logs/source-build/binlogs \;"
64+
$(docker.runInSourceBuild) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/source-build/logs; find ./bin/logs -name '*.log' -exec cp {} /logs/source-build/logs \;"
65+
displayName: Copy source-build logs
66+
condition: always()
67+
continueOnError: true
68+
69+
# run smoke tests
70+
- script: $(docker.runInSourceBuild) $(imageName) ./build.sh /t:RunSmokeTest /p:Configuration=$(buildConfiguration) /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
71+
displayName: Run smoke-test
72+
73+
# copy smoke test logs to working directory
74+
- script: $(docker.runInSourceBuild) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/source-build/smoke-test; find ./testing-smoke -name '*.log' -exec cp {} /logs/source-build/smoke-test \;"
75+
displayName: Copy smoke-test logs
76+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], false))
77+
continueOnError: true
78+
79+
# create tarball
80+
- script: $(docker.runInSourceBuild) -v $(rootDirectory)/sb/tarball:/tb $(imageName) ./build-source-tarball.sh /tb/$(tarballName) --skip-build
81+
displayName: Create tarball
82+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
83+
84+
# tar the tarball directory into the drop directory
85+
- script: $(docker.runInSourceBuild) -v $(rootDirectory)/sb/tarball:/tb -v $(dropDirectory):/drop $(imageName) tar -zcvf /drop/$(tarballName).tar.gz /tb/$(tarballName)
86+
displayName: Copy tarball to output
87+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
88+
89+
# build tarball
90+
- script: $(docker.runInTarball) --network='none' $(imageName) ./build.sh /p:Configuration=$(buildConfiguration) $(buildLoggingOptions)
91+
displayName: Build tarball
92+
timeoutInMinutes: 90
93+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
94+
95+
# run smoke tests
96+
- script: $(docker.runInTarball) $(imageName) ./smoke-test.sh --minimal --projectOutput --configuration $(buildConfiguration) --prodConBlobFeedUrl ''
97+
displayName: Run smoke-test in tarball
98+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
99+
100+
# copy all tarball logs to working directory
101+
- script: |
102+
set -x
103+
$(docker.runInTarball) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/tarball/binlogs; find . -name '*.binlog' -exec cp {} /logs/tarball/binlogs \;"
104+
$(docker.runInTarball) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/tarball/logs; find ./bin/logs -name '*.log' -exec cp {} /logs/tarball/logs \;"
105+
$(docker.runInTarball) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/tarball/smoke-test; find ./testing-smoke -name '*.log' -exec cp {} /logs/tarball/smoke-test \;"
106+
# Copy prebuilt report
107+
$(docker.runInSourceBuild) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/prebuilt-report/online; cp -r ./bin/prebuilt-report/* /logs/prebuilt-report/online"
108+
$(docker.runInTarball) -v $(dropDirectory)/logs:/logs $(imageName) /bin/bash -c "mkdir -p /logs/prebuilt-report/offline; cp -r ./bin/prebuilt-report/* /logs/prebuilt-report/offline"
109+
displayName: Copy tarball logs
110+
condition: eq(variables['buildOfflineTarball'], true)
111+
continueOnError: true
112+
113+
# copy artifacts to staging - Copy to VSTS owned folder is done outside of docker so copied files have correct ownership so VSTS can clean them up later.
114+
- task: CopyFiles@2
115+
condition: always()
116+
continueOnError: true
117+
inputs:
118+
sourceFolder: $(stagingDirectory)
119+
targetFolder: $(Build.ArtifactStagingDirectory)
120+
121+
# publish artifacts
122+
- task: PublishBuildArtifacts@1
123+
displayName: Publish artifacts
124+
condition: always()
125+
continueOnError: true
126+
inputs:
127+
PathtoPublish: $(Build.ArtifactStagingDirectory)/drop
128+
ArtifactName: drop
129+
ArtifactType: Container
130+
131+
- template: ./.vsts.pipelines/steps/docker-cleanup-linux.yml

‎.vsts.pipelines/builds/vsts_prodcon_update.yml

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
phases:
2+
- phase: vsts_prodcon_update
3+
variables:
4+
# autoUpdate.skipBuildToolsUpdate: (User input: Should the BuildTools update be skipped? This is useful to isolate the update to ProdCon.)
5+
# pb.feed.buildId: (User input: What build id is the prodcon blob feed from? This shows up in the commit.)
6+
# pb.feed.indexJsonUrl: (User input: Where is the prodcon blob feed?)
7+
# pb.feed.replaceNew: (User input for ProdConBlobFeedReplaceNew.)
8+
# pb.feed.replaceOld: (User input for ProdConBlobFeedReplaceOld.)
9+
manifestDir: $(repo.orchestrationUtilities.dir)\bin\obj\manifest-temp
10+
manifestFile: $(manifestDir)\build.xml
11+
repo.orchestrationUtilities.dir: $(Build.BinariesDirectory)\utilities
12+
repo.orchestrationUtilities.url: https://usernameplaceholder:$(user.PAT)@devdiv.visualstudio.com/DefaultCollection/DevDiv/_git/DotNet-Orchestration-Utilities
13+
repo.sourceBuild.url: https://usernameplaceholder:$(user.PAT)@devdiv.visualstudio.com/DefaultCollection/DevDiv/_git/DotNet-Source-Build-Trusted
14+
15+
queue:
16+
name: DotNet-Build
17+
demands: agent.os -equals windows_nt
18+
timeoutInMinutes: 180
19+
20+
steps:
21+
- powershell: |
22+
$prefix = "refs/heads/"
23+
$branch = "$(Build.SourceBranch)"
24+
$branchName = $branch
25+
if ($branchName.StartsWith($prefix))
26+
{
27+
$branchName = $branchName.Substring($prefix.Length)
28+
}
29+
Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName"
30+
Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName"
31+
displayName: Find true SourceBranchName
32+
33+
- powershell: |
34+
# Ignore failure for the first command. It will intentionally fail if the commit is only
35+
# available in VSTS. "submodule update --init" is the simplest way to set up the submodule
36+
# directory. ("submodule init" only sets up .git/config, not the e.g. src/coreclr/.git and
37+
# .git/modules/src/coreclr/ directories.)
38+
git submodule update --init --recursive
39+
# This fetches old commits (allowing the next update to succeed) and populates the git repo
40+
# with the potential commits that may be upgraded to.
41+
./fetch-vsts-commits.ps1 $(user.PAT)
42+
git submodule update --init --recursive
43+
displayName: Fetch internal vsts commits
44+
45+
- powershell: |
46+
# Ensure no straggling artifacts could interfere with manifest generation.
47+
rm -Recurse -Force -ErrorAction Ignore "$(repo.orchestrationUtilities.dir)"
48+
git clone "$(repo.orchestrationUtilities.url)" "$(repo.orchestrationUtilities.dir)"
49+
displayName: Initialize DotNet-Orchestration-Utilities
50+
51+
- powershell: |
52+
.\init-tools.cmd
53+
Tools\dotnetcli\dotnet msbuild .\src\publish.proj `
54+
/t:CreateFeedOrchestratedBuildManifest `
55+
/p:ExpectedFeedUrl=$(pb.feed.indexJsonUrl) `
56+
/p:AccountKey=$(pb.feed.accountKey) `
57+
/p:ManifestName=cli `
58+
/p:ManifestBuildId=$(pb.feed.buildId) `
59+
/v:N /flp:v=Diag
60+
workingDirectory: $(repo.orchestrationUtilities.dir)
61+
displayName: Create orchestrated build manifest
62+
63+
- powershell: |
64+
.\build.cmd /t:InitBuild /p:SkipPatches=true
65+
.\build.cmd `
66+
/t:UpdateDependencies `
67+
/p:SkipBuildToolsUpdate=$(autoUpdate.skipBuildToolsUpdate) `
68+
/p:UpdateFromManifestFile=$(manifestFile) `
69+
/p:ProdConBlobFeedReplaceOld=$(pb.feed.replaceOld) `
70+
/p:ProdConBlobFeedReplaceNew=$(pb.feed.replaceNew) `
71+
/clp:v=Normal
72+
displayName: Perform auto-update changes
73+
74+
- powershell: |
75+
git `
76+
-c user.name=dotnet-maestro-bot `
77+
-c "[email protected]" `
78+
commit -a -m "Update to ProdCon $(pb.feed.buildId)"
79+
git push -f $(repo.sourceBuild.url) HEAD:refs/heads/auto-update/$(FullBranchName)
80+
displayName: Push auto-update commit
81+
82+
- task: CopyFiles@2
83+
condition: always()
84+
continueOnError: true
85+
inputs:
86+
sourceFolder: $(manifestDir)
87+
targetFolder: $(Build.ArtifactStagingDirectory)\manifestDir
88+
89+
- task: PublishBuildArtifacts@1
90+
displayName: Publish artifacts
91+
condition: always()
92+
continueOnError: true
93+
inputs:
94+
PathtoPublish: $(Build.ArtifactStagingDirectory)
95+
ArtifactName: artifacts
96+
ArtifactType: Container
97+
98+
- powershell: rm -Recurse -Force "$(repo.orchestrationUtilities.dir)"
99+
condition: always()
100+
continueOnError: true
101+
displayName: Clean up DotNet-Orchestration-Utilities

‎.vsts.pipelines/steps/docker-cleanup-linux.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
steps:
2+
- script: docker system prune -a -f --volumes
3+
displayName: Cleanup Docker
4+
condition: always()
5+
continueOnError: true

‎build-source-tarball.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,20 @@ mkdir -p $TARBALL_ROOT/prebuilt/nuget-packages
8383
find $SCRIPT_ROOT/packages -name '*.nupkg' -exec cp {} $TARBALL_ROOT/prebuilt/nuget-packages/ \;
8484
find $SCRIPT_ROOT/bin/obj/x64/Release/nuget-packages -name '*.nupkg' -exec cp {} $TARBALL_ROOT/prebuilt/nuget-packages/ \;
8585

86+
if [ -e $SCRIPT_ROOT/testing-smoke/smoke-test-packages ]; then
87+
cp -rf $SCRIPT_ROOT/testing-smoke/smoke-test-packages $TARBALL_ROOT/prebuilt
88+
fi
89+
8690
echo 'Removing source-built packages from tarball prebuilts...'
8791

8892
for built_package in $(find $SCRIPT_ROOT/bin/obj/x64/Release/blob-feed/packages/ -name '*.nupkg' | tr '[:upper:]' '[:lower:]')
8993
do
9094
if [ -e $TARBALL_ROOT/prebuilt/nuget-packages/$(basename $built_package) ]; then
9195
rm $TARBALL_ROOT/prebuilt/nuget-packages/$(basename $built_package)
9296
fi
97+
if [ -e $TARBALL_ROOT/prebuilt/smoke-test-packages/$(basename $built_package) ]; then
98+
rm $TARBALL_ROOT/prebuilt/smoke-test-packages/$(basename $built_package)
99+
fi
93100
done
94101

95102
if [ -z "${SOURCE_BUILD_SKIP_PREBUILT_REPORT:-}" ]; then

‎build.proj

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,19 @@
5151
<MSBuild Projects="repos\$(RootRepo).proj" Targets="ReportPrebuiltUsage" />
5252
</Target>
5353

54+
<Target Name="RunSmokeTest" DependsOnTargets="GetProdConBlobFeedUrl">
55+
<!--
56+
Pass prodConBlobFeedUrl via EnvironmentVariables because it has '//' in it, which is
57+
translated into '/' if it's passed in the Command arg.
58+
This is also a problem when passing CLI feeds: https://github.com/dotnet/source-build/issues/561
59+
-->
60+
<Exec Command="./smoke-test.sh --minimal --projectOutput --configuration $(Configuration) --archiveRestoredPackages"
61+
EnvironmentVariables="prodConBlobFeedUrl=$(ProdConBlobFeedUrl)" />
62+
</Target>
63+
5464
<Import Project="$(ProjectDir)dependencies.targets" />
5565

5666
<Import Project="$(ToolsDir)VersionTools.targets" />
67+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))/dir.targets" />
5768

5869
</Project>

‎dependencies.props

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,19 +38,17 @@
3838
<RawVersionsBaseUrl>https://raw.githubusercontent.com/dotnet/versions</RawVersionsBaseUrl>
3939
</DependencyBuildInfo>
4040

41-
<DependencyInfo Include="ProdCon">
41+
<DependencyInfo Include="ProdCon" Condition="'$(UpdateFromManifestFile)' == ''">
4242
<DependencyType>Orchestrated build</DependencyType>
4343
<BasePath>build-info/dotnet/product/cli/release/2.1.1</BasePath>
4444
<CurrentRef>$(ProdConCurrentRef)</CurrentRef>
4545
<VersionsRepoOwner>dotnet</VersionsRepoOwner>
4646
<VersionsRepo>versions</VersionsRepo>
4747
</DependencyInfo>
48-
49-
<UpdateStep Include="ProdCon">
50-
<UpdaterType>Orchestrated blob feed attribute</UpdaterType>
51-
<SingleLineFile>$(MSBuildThisFileDirectory)ProdConFeed.txt</SingleLineFile>
52-
<AttributeName>Url</AttributeName>
53-
</UpdateStep>
48+
<DependencyInfo Include="ProdCon" Condition="'$(UpdateFromManifestFile)' != ''">
49+
<DependencyType>Orchestrated build file</DependencyType>
50+
<Path>$(UpdateFromManifestFile)</Path>
51+
</DependencyInfo>
5452
</ItemGroup>
5553

5654
<ItemGroup Condition="'$(SkipBuildToolsUpdate)' != 'true'">

‎dependencies.targets

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,16 @@
8484
UpdaterType="Build attribute from orchestrated build"
8585
AttributeName="BuildId"
8686
Path="%(RepoProjectWithProperties.Identity)"
87-
ElementName="OfficialBuildId" />
87+
ElementName="OfficialBuildId"
88+
SkipIfNoReplacementFound="true" />
89+
90+
<UpdateStep
91+
Include="ProdCon"
92+
UpdaterType="Orchestrated blob feed attribute"
93+
SingleLineFile="$(MSBuildThisFileDirectory)ProdConFeed.txt"
94+
AttributeName="Url"
95+
ReplacementSubstituteOld="$(ProdConBlobFeedReplaceOld)"
96+
ReplacementSubstituteNew="$(ProdConBlobFeedReplaceNew)" />
8897
</ItemGroup>
8998
</Target>
9099
</Project>

‎dir.targets

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
4+
<Target Name="GetProdConBlobFeedUrl">
5+
<PropertyGroup>
6+
<ProdConBlobFeedUrl>$([System.IO.File]::ReadAllText('$(ProdConFeedPath)').Trim())</ProdConBlobFeedUrl>
7+
<ProdConBlobFeedUrl Condition="'$(ProdConBlobFeedUrlPrefix)' != ''">$(ProdConBlobFeedUrl.Replace('https://dotnetfeed.blob.core.windows.net/', '$(ProdConBlobFeedUrlPrefix)'))</ProdConBlobFeedUrl>
8+
</PropertyGroup>
9+
</Target>
10+
</Project>

‎fetch-vsts-commits.ps1

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
param (
2+
[Parameter(Mandatory=$true)][string]$pat,
3+
[string]$remote_ref_name = "vsts"
4+
)
5+
6+
function get_vsts_url(
7+
[Parameter(Mandatory=$true)][string]$name,
8+
[string]$instance = 'devdiv.visualstudio.com',
9+
[string]$project = 'DevDiv')
10+
{
11+
return "https://usernameplaceholder:${pat}@${instance}/${project}/_git/${name}/"
12+
}
13+
14+
function fetch(
15+
[Parameter(Mandatory=$true)][string]$name,
16+
[Parameter(Mandatory=$true)][string]$remote)
17+
{
18+
$cmd = @(
19+
"-C", "src/$name"
20+
# Disable credential manager if one is specified. Always use manual PAT.
21+
"-c", "credential.helper="
22+
"fetch"
23+
$remote
24+
"+refs/heads/*:refs/remotes/${remote_ref_name}/*"
25+
)
26+
echo "Fetching commits using: & git $cmd"
27+
& git @cmd
28+
}
29+
30+
function fetch_vsts(
31+
[Parameter(Mandatory=$true)][string]$name,
32+
[string]$url = (get_vsts_url "DotNet-$name-Trusted"))
33+
{
34+
fetch "$name" "$url"
35+
}
36+
37+
fetch_vsts cli
38+
fetch_vsts core-setup
39+
fetch_vsts coreclr
40+
fetch_vsts corefx

0 commit comments

Comments
 (0)