Skip to content
This repository was archived by the owner on Feb 23, 2021. It is now read-only.

Commit db4edc1

Browse files
pranavkmpranavkm
committed
Modify Coherence to publish partner packages to ext directory.
1 parent 7fbcf1e commit db4edc1

File tree

11 files changed

+373
-323
lines changed

11 files changed

+373
-323
lines changed

‎makefile.shade‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,33 +18,44 @@ default PACKAGE_DROP_SHARE='${Environment.GetEnvironmentVariable("PACKAGE_DROP_S
1818
default NUGET_PUBLISH_FEED='${Environment.GetEnvironmentVariable("NUGET_PUBLISH_FEED")}'
1919
default APIKEY='${Environment.GetEnvironmentVariable("APIKEY")}'
2020
default NATIVE_CACHE_VERSION = '${Environment.GetEnvironmentVariable("DOTNET_BUILD_VERSION")}'
21+
default UNIVERSECOHERENCE_BUILD = '${Environment.GetEnvironmentVariable("COHERENCE_UNIVERSECOHERENCE_BUILD_NUMBER")}'
22+
default CORECLR_BUILD = '${Environment.GetEnvironmentVariable("COHERENCE_CORECLR_BUILD_NUMBER")}'
23+
default DISABLE_PRODUCT_SANITYCHECK = '${Environment.GetEnvironmentVariable("DISABLE_COHERENCE_CHECK") == "true"}'
24+
default DISABLE_PARTNER_SANITYCHECK = '${Environment.GetEnvironmentVariable("DISABLE_PARTNER_COHERENCE_CHECK") == "true"}'
2125

2226
use-standard-lifecycle
2327

2428
#package-restore target='initialize'
2529
exec program='dotnet' commandline='restore --infer-runtimes src'
2630
exec program='dotnet' commandline='restore project.json --configfile ${BASE_DIR}/NuGet.config --packages ${TOOLS_DOWNLOAD_PATH} -v Minimal'
2731

28-
#build-project target='compile'
29-
exec program='dotnet' commandline='build -f net451' workingdir='${COHERENCE_BUILD_PROJ}'
30-
3132
#copy-bits target='package'
33+
var args = '--drop-folder ${BUILD_SHARE} --build-branch ${BUILD_BRANCH} --output-path ${TARGET_DIR} --universecoherence-build ${UNIVERSECOHERENCE_BUILD} --coreclr-build ${CORECLR_BUILD}'
3234
@{
33-
if (String.IsNullOrEmpty(BUILD_SHARE))
34-
{
35-
Log.Warn("BUILD_SHARE environment varible is not set");
36-
Environment.Exit(-1);
37-
return;
38-
}
39-
}
40-
41-
var args = '--drop-folder ${BUILD_SHARE} --build-branch ${BUILD_BRANCH} --output-path ${TARGET_DIR} --nuget-publish-feed ${NUGET_PUBLISH_FEED} --apikey ${APIKEY}'
35+
if (DISABLE_PARTNER_SANITYCHECK)
36+
{
37+
args += " --disable-partner-package-verification";
38+
}
39+
40+
if (DISABLE_PRODUCT_SANITYCHECK)
41+
{
42+
args += " --disable-product-package-verification";
43+
}
44+
45+
if (!string.IsNullOrEmpty(NUGET_PUBLISH_FEED))
46+
{
47+
args += string.Format("--nuget-publish-feed {0} --api-key {1}", NUGET_PUBLISH_FEED, APIKEY);
48+
}
49+
}
50+
4251
exec program='dotnet' commandline='run -f net451 -- ${args}' workingdir='${COHERENCE_BUILD_PROJ}'
4352

53+
#split-packages target='package'
4454
var splitPackagesExe = '${Directory.GetFiles(TOOLS_DOWNLOAD_PATH, "SplitPackages.exe", SearchOption.AllDirectories).First()}'
4555
var splitArgs = '--source ${BUILD_DIR} --csv ${SPLIT_CSV_PATH} --destination ${TARGET_DIR}'
4656
exec program='${splitPackagesExe}' commandline='${splitArgs}' workingdir='${COHERENCE_BUILD_PROJ}'
4757

58+
#package-dependencies target='package'
4859
var dependenciesPackagerExe = '${Directory.GetFiles(TOOLS_DOWNLOAD_PATH, "DependenciesPackager.exe", SearchOption.AllDirectories).First()}'
4960
var dependenciesPackagerArgs = '--project ${TARGET_DIR}\ship\project.json --diagnostics-project ${TARGET_DIR}\ship\noimports.project.json --sources ${TARGET_DIR}\ship --sources ${TARGET_DIR}\ship-ext --sources ${TARGET_DIR}\ext --fallback https://api.nuget.org/v3/index.json --destination ${TARGET_DIR} --version ${NATIVE_CACHE_VERSION}'
5061
exec program='${dependenciesPackagerExe}' commandline='${dependenciesPackagerArgs}' workingdir='${COHERENCE_BUILD_PROJ}'

‎packages/packages.csv‎

Lines changed: 1 addition & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,9 @@
11
Package,Category
2-
dnx-clr-win-x64,noship
3-
dnx-clr-win-x86,noship
4-
dnx-coreclr-darwin-x64,noship
5-
dnx-coreclr-linux-x64,noship
6-
dnx-coreclr-win-arm,noship
7-
dnx-coreclr-win-x64,noship
8-
dnx-coreclr-win-x86,noship
9-
dnx-mono,noship
102
Microsoft.AspNetCore.Razor.Tools,ship
113
Microsoft.VisualStudio.Web.CodeGeneration.Tools,ship
12-
dotnet-test-xunit,ext
134
Microsoft.Extensions.SecretManager.Tools,ship
145
Microsoft.DotNet.Watcher.Tools,ship
6+
Microsoft.DotNet.Watcher.Core,ship
157
Libuv,ship-ext
168
Microsoft.AspNet.SignalR.Client.Cpp.v120.WinDesktop,noship
179
Microsoft.AspNet.SignalR.Client.Cpp.v140.WinDesktop,noship
@@ -53,9 +45,6 @@ Microsoft.AspNetCore.Http.Features,ship
5345
Microsoft.AspNetCore.HttpOverrides,ship
5446
Microsoft.AspNetCore.Identity,ship
5547
Microsoft.AspNetCore.Identity.EntityFrameworkCore,ship
56-
Microsoft.AspNetCore.Internal.libuv-Darwin,noship
57-
Microsoft.AspNetCore.Internal.libuv-Linux,noship
58-
Microsoft.AspNetCore.Internal.libuv-Windows,noship
5948
Microsoft.AspNetCore.JsonPatch,ship
6049
Microsoft.AspNetCore.Localization,ship
6150
Microsoft.AspNetCore.MiddlewareAnalysis,ship
@@ -101,13 +90,7 @@ Microsoft.AspNetCore.WebSockets.Client,noship
10190
Microsoft.AspNetCore.WebSockets.Protocol,ship
10291
Microsoft.AspNetCore.WebSockets.Server,ship
10392
Microsoft.AspNetCore.WebUtilities,ship
104-
Microsoft.CodeAnalysis.*,ext
105-
Microsoft.CSharp,ext
10693
Microsoft.Data.Sqlite,ship
107-
Microsoft.Dnx.*,noship
108-
Microsoft.Dnx.Compilation.CSharp.Abstractions,noship
109-
Microsoft.Dnx.Compilation.CSharp.Common,noship
110-
Microsoft.DotNet.*,ext
11194
Microsoft.EntityFrameworkCore,ship
11295
Microsoft.EntityFrameworkCore.InMemory,ship
11396
Microsoft.EntityFrameworkCore.Relational,ship
@@ -136,8 +119,6 @@ Microsoft.VisualStudio.Web.CodeGeneration.Templating,ship
136119
Microsoft.VisualStudio.Web.CodeGeneration.Utils,ship
137120
Microsoft.VisualStudio.Web.CodeGenerators.Mvc,ship
138121
Microsoft.Extensions.CommandLineUtils,ship
139-
Microsoft.Extensions.CommandLineUtils.Sources,noship
140-
Microsoft.Extensions.CompilationAbstractions,noship
141122
Microsoft.Extensions.Configuration,ship
142123
Microsoft.Extensions.Configuration.Abstractions,ship
143124
Microsoft.Extensions.Configuration.Binder,ship
@@ -152,7 +133,6 @@ Microsoft.Extensions.CopyOnWriteDictionary.Sources,noship
152133
Microsoft.Extensions.DependencyInjection,ship
153134
Microsoft.Extensions.DependencyInjection.Abstractions,ship
154135
Microsoft.Extensions.DependencyInjection.Specification.Tests,ship
155-
Microsoft.Extensions.DependencyModel,ship
156136
Microsoft.Extensions.DiagnosticAdapter,ship
157137
Microsoft.Extensions.DotnetToolDispatcher.Sources,noship
158138
Microsoft.Extensions.FileProviders.Abstractions,ship
@@ -163,7 +143,6 @@ Microsoft.Extensions.FileProviders.Sources,noship
163143
Microsoft.Extensions.FileSystemGlobbing,ship
164144
Microsoft.Extensions.Globalization.CultureInfoCache,ship
165145
Microsoft.Extensions.HashCodeCombiner.Sources,noship
166-
Microsoft.Extensions.JsonParser.Sources,noship
167146
Microsoft.Extensions.Localization,ship
168147
Microsoft.Extensions.Localization.Abstractions,ship
169148
Microsoft.Extensions.Logging,ship
@@ -178,28 +157,16 @@ Microsoft.Extensions.ObjectPool,ship
178157
Microsoft.Extensions.Options,ship
179158
Microsoft.Extensions.Options.ConfigurationExtensions,ship
180159
Microsoft.Extensions.PlatformAbstractions,ship
181-
Microsoft.Extensions.PlatformAbstractions.Dnx,noship
182160
Microsoft.Extensions.Primitives,ship
183161
Microsoft.Extensions.Process.Sources,noship
184162
Microsoft.Extensions.PropertyActivator.Sources,noship
185163
Microsoft.Extensions.PropertyHelper.Sources,noship
186164
Microsoft.Extensions.SecurityHelper.Sources,noship
187-
Microsoft.Extensions.Testing.Abstractions,ext
188165
Microsoft.Extensions.TypeNameHelper.Sources,noship
189166
Microsoft.Extensions.WebEncoders,ship
190-
Microsoft.IdentityModel.*,ext
191167
Microsoft.Net.Http.Headers,ship
192168
Microsoft.Net.Http.Server,ship
193169
Microsoft.Net.WebSockets.Server,ship
194-
Microsoft.NETCore.*,ext
195170
Microsoft.Owin.Security.Interop,ship
196-
Microsoft.TargetingPack.*,ext
197-
Microsoft.VisualBasic,ext
198171
Microsoft.VisualStudio.Web.BrowserLink.Loader,ship
199-
Microsoft.Win32.*,ext
200-
NETStandard.Library,ext
201-
NuGet.*,ext
202172
PageGenerator,noship
203-
runtime.*,ext
204-
System.*,ext
205-
xunit.runner.*,noship

‎src/CoherenceBuild/CoherenceBuild.cs‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
using System.Collections.Concurrent;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Threading.Tasks;
6+
using NuGet.Packaging;
7+
8+
namespace CoherenceBuild
9+
{
10+
public class CoherenceBuild
11+
{
12+
private const int SuccessExitCode = 0;
13+
private const int FailureExitCode = 0;
14+
15+
private List<RepositoryInfo> _reposToProcess;
16+
private readonly string _dropFolder;
17+
private readonly string _buildBranch;
18+
private readonly string _outputPath;
19+
private readonly string _nugetPublishFeed;
20+
private readonly string _apiKey;
21+
22+
public CoherenceBuild(
23+
List<RepositoryInfo> reposToProcess,
24+
string dropFolder,
25+
string buildBranch,
26+
string outputPath,
27+
string nugetPublishFeed,
28+
string apiKey)
29+
{
30+
_reposToProcess = reposToProcess;
31+
_dropFolder = dropFolder;
32+
_buildBranch = buildBranch;
33+
_outputPath = outputPath;
34+
_nugetPublishFeed = nugetPublishFeed;
35+
_apiKey = apiKey;
36+
}
37+
38+
public CoherenceVerifyBehavior VerifyBehavior { get; set; } = CoherenceVerifyBehavior.All;
39+
40+
public int Execute()
41+
{
42+
if (!Directory.Exists(_dropFolder))
43+
{
44+
Log.WriteError($"Drop share {_dropFolder} does not exist");
45+
return FailureExitCode;
46+
}
47+
48+
Directory.CreateDirectory(_outputPath);
49+
var processedPackages = new ConcurrentBag<PackageInfo>();
50+
foreach (var repo in _reposToProcess)
51+
{
52+
Program.Retry(() =>
53+
{
54+
ProcessRepo(processedPackages, repo);
55+
});
56+
}
57+
58+
var coherenceVerify = new CoherenceVerifier(processedPackages, VerifyBehavior);
59+
if (!coherenceVerify.VerifyAll())
60+
{
61+
return FailureExitCode;
62+
}
63+
64+
if (!string.IsNullOrEmpty(_nugetPublishFeed))
65+
{
66+
PackagePublisher.PublishToFeed(processedPackages, _nugetPublishFeed, _apiKey);
67+
}
68+
69+
var expandDirectory = Path.Combine(_outputPath, "packages-expanded");
70+
Log.WriteInformation($"Expanding packages to {expandDirectory}");
71+
Directory.CreateDirectory(expandDirectory);
72+
PackagePublisher.ExpandPackageFiles(processedPackages, expandDirectory);
73+
74+
return SuccessExitCode;
75+
}
76+
77+
private void ProcessRepo(ConcurrentBag<PackageInfo> processedPackages, RepositoryInfo repo)
78+
{
79+
var repoDirectory = Path.Combine(_dropFolder, repo.Name, _buildBranch);
80+
if (string.IsNullOrEmpty(repo.BuildNumber))
81+
{
82+
repo.BuildNumber = FindLatest(repoDirectory);
83+
}
84+
85+
repoDirectory = Path.Combine(repoDirectory, repo.BuildNumber);
86+
foreach (var dependency in repo.FileSystemDependencies)
87+
{
88+
dependency.Copy(repoDirectory, _outputPath);
89+
}
90+
91+
var packageSourceDir = Path.Combine(repoDirectory, repo.PackageSourceDir);
92+
93+
var packageTargetDir = Path.Combine(_outputPath, repo.PackageDestinationDir);
94+
var symbolsTargetDir = Path.Combine(_outputPath, "symbols");
95+
Directory.CreateDirectory(symbolsTargetDir);
96+
Directory.CreateDirectory(packageTargetDir);
97+
98+
Parallel.ForEach(Directory.GetFiles(packageSourceDir, "*.nupkg"), packagePath =>
99+
{
100+
var packageFileName = Path.GetFileName(packagePath);
101+
if (packagePath.EndsWith(".symbols.nupkg"))
102+
{
103+
var targetPath = Path.Combine(symbolsTargetDir, packageFileName);
104+
File.Copy(packagePath, targetPath, overwrite: true);
105+
return;
106+
}
107+
108+
var packageInfo = new PackageInfo
109+
{
110+
IsPartnerPackage = repo.Name == "CoreCLR",
111+
PackagePath = Path.Combine(packageTargetDir, packageFileName),
112+
};
113+
114+
using (var fileStream = File.OpenRead(packagePath))
115+
using (var reader = new PackageArchiveReader(fileStream))
116+
{
117+
packageInfo.Identity = reader.GetIdentity();
118+
packageInfo.PackageDependencyGroups = reader.GetPackageDependencies();
119+
}
120+
121+
if (repo.PackagesToSkip.Contains(packageInfo.Identity.Id))
122+
{
123+
Log.WriteInformation($"Skipping package {packagePath}");
124+
return;
125+
}
126+
127+
File.Copy(packagePath, packageInfo.PackagePath, overwrite: true);
128+
processedPackages.Add(packageInfo);
129+
});
130+
}
131+
132+
private void WriteReposUsedFile()
133+
{
134+
var filePath = Path.Combine(_outputPath, "packages-sources");
135+
var fileContent = string.Join("\n", _reposToProcess.Select(r => $"{r.Name}: {r.BuildNumber}"));
136+
File.WriteAllText(filePath, fileContent);
137+
}
138+
139+
private static string FindLatest(string repoDirectory)
140+
{
141+
if (!Directory.Exists(repoDirectory))
142+
{
143+
return null;
144+
}
145+
146+
return new DirectoryInfo(repoDirectory)
147+
.EnumerateDirectories()
148+
.Select(d =>
149+
{
150+
int buildNumber;
151+
if (!int.TryParse(d.Name, out buildNumber))
152+
{
153+
buildNumber = int.MinValue;
154+
}
155+
156+
return new
157+
{
158+
DirectoryInfo = d,
159+
BuildNumber = buildNumber
160+
};
161+
})
162+
.OrderByDescending(r => r.BuildNumber)
163+
.Select(r => r.DirectoryInfo.Name)
164+
.FirstOrDefault();
165+
}
166+
167+
}
168+
}

0 commit comments

Comments
 (0)