Skip to content

Commit 80ab2db

Browse files
loic-sharmatomzo
loic-sharma
authored and
tomzo
committed
Add file storage tests, refactor
cherry-pick of upstream tests 5324d3b Then - added implementations from upstream - mixed tests with ours
1 parent e33eead commit 80ab2db

File tree

10 files changed

+576
-379
lines changed

10 files changed

+576
-379
lines changed

‎src/BaGet.Azure/BlobPackageStorageService.cs‎

Lines changed: 6 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.IO;
33
using System.Threading;
44
using System.Threading.Tasks;
5+
using BaGet.Core.Entities;
56
using BaGet.Core.Extensions;
67
using BaGet.Core.Services;
78
using Microsoft.WindowsAzure.Storage;
@@ -25,36 +26,6 @@ public BlobPackageStorageService(CloudBlobContainer container)
2526
_container = container ?? throw new ArgumentNullException(nameof(container));
2627
}
2728

28-
public async Task SavePackageStreamAsync(
29-
PackageArchiveReader package,
30-
Stream packageStream,
31-
CancellationToken cancellationToken)
32-
{
33-
var identity = await package.GetIdentityAsync(cancellationToken);
34-
var lowercasedId = identity.Id.ToLowerInvariant();
35-
var lowercasedNormalizedVersion = identity.Version.ToNormalizedString().ToLowerInvariant();
36-
37-
var packageBlob = GetPackageBlob(lowercasedId, lowercasedNormalizedVersion);
38-
var nuspecBlob = GetNuspecBlob(lowercasedId, lowercasedNormalizedVersion);
39-
var readmeBlob = GetReadmeBlob(lowercasedId, lowercasedNormalizedVersion);
40-
41-
// Save the package's nupkg
42-
packageStream.Seek(0, SeekOrigin.Begin);
43-
await UploadBlobAsync(packageBlob, packageStream, PackageContentType);
44-
45-
// Save the package's nuspec
46-
using (var nuspecStream = await package.GetNuspecAsync(cancellationToken))
47-
{
48-
await UploadBlobAsync(nuspecBlob, nuspecStream, TextContentType);
49-
}
50-
51-
// Save the package's reamde
52-
using (var readmeStream = package.GetReadme())
53-
{
54-
await UploadBlobAsync(readmeBlob, readmeStream, TextContentType);
55-
}
56-
}
57-
5829
public async Task DeleteAsync(PackageIdentity id)
5930
{
6031
var lowercasedId = id.Id.ToLowerInvariant();
@@ -154,5 +125,10 @@ private string ReadmePath(string lowercasedId, string lowercasedNormalizedVersio
154125
lowercasedNormalizedVersion,
155126
"readme");
156127
}
128+
129+
public Task SavePackageContentAsync(Package package, Stream packageStream, Stream nuspecStream, Stream readmeStream, CancellationToken cancellationToken)
130+
{
131+
throw new NotImplementedException();
132+
}
157133
}
158134
}

‎src/BaGet.Core/Extensions/PackageArchiveReaderExtensions.cs‎

Lines changed: 114 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
using System.Collections.Generic;
33
using System.IO;
44
using System.Linq;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using BaGet.Core.Entities;
58
using NuGet.Packaging;
69

710
namespace BaGet.Core.Extensions
@@ -21,8 +24,9 @@ public static class PackageArchiveReaderExtensions
2124
public static bool HasReadme(this PackageArchiveReader package)
2225
=> package.GetFiles().Any(ReadmeFileNames.Contains);
2326

24-
// TODO: This should be async and accept a CancellationToken.
25-
public static Stream GetReadme(this PackageArchiveReader package)
27+
public async static Task<Stream> GetReadmeAsync(
28+
this PackageArchiveReader package,
29+
CancellationToken cancellationToken)
2630
{
2731
var packageFiles = package.GetFiles();
2832

@@ -32,11 +36,117 @@ public static Stream GetReadme(this PackageArchiveReader package)
3236

3337
if (readmePath != null)
3438
{
35-
return package.GetStream(readmePath);
39+
return await package.GetStreamAsync(readmePath, cancellationToken);
3640
}
3741
}
3842

39-
return Stream.Null;
43+
throw new InvalidOperationException("Package does not have a readme!");
44+
}
45+
46+
public static Package GetPackageMetadata(this PackageArchiveReader packageReader)
47+
{
48+
var nuspec = packageReader.NuspecReader;
49+
50+
(var repositoryUri, var repositoryType) = GetRepositoryMetadata(nuspec);
51+
52+
return new Package
53+
{
54+
Id = nuspec.GetId(),
55+
Version = nuspec.GetVersion(),
56+
Authors = ParseAuthors(nuspec.GetAuthors()),
57+
Description = nuspec.GetDescription(),
58+
HasReadme = packageReader.HasReadme(),
59+
Language = nuspec.GetLanguage() ?? string.Empty,
60+
Listed = true,
61+
MinClientVersion = nuspec.GetMinClientVersion()?.ToNormalizedString() ?? string.Empty,
62+
Published = DateTime.UtcNow,
63+
RequireLicenseAcceptance = nuspec.GetRequireLicenseAcceptance(),
64+
Summary = nuspec.GetSummary(),
65+
Title = nuspec.GetTitle(),
66+
IconUrl = ParseUri(nuspec.GetIconUrl()),
67+
LicenseUrl = ParseUri(nuspec.GetLicenseUrl()),
68+
ProjectUrl = ParseUri(nuspec.GetProjectUrl()),
69+
RepositoryUrl = repositoryUri,
70+
RepositoryType = repositoryType,
71+
Dependencies = GetDependencies(nuspec),
72+
Tags = ParseTags(nuspec.GetTags())
73+
};
74+
}
75+
76+
private static Uri ParseUri(string uriString)
77+
{
78+
if (string.IsNullOrEmpty(uriString)) return null;
79+
80+
return new Uri(uriString);
81+
}
82+
83+
private static string[] ParseAuthors(string authors)
84+
{
85+
if (string.IsNullOrEmpty(authors)) return new string[0];
86+
87+
return authors.Split(new[] { ',', ';', '\t', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
88+
}
89+
90+
private static string[] ParseTags(string tags)
91+
{
92+
if (string.IsNullOrEmpty(tags)) return new string[0];
93+
94+
return tags.Split(new[] { ',', ';', ' ', '\t', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
95+
}
96+
97+
private static (Uri repositoryUrl, string repositoryType) GetRepositoryMetadata(NuspecReader nuspec)
98+
{
99+
var repository = nuspec.GetRepositoryMetadata();
100+
101+
if (string.IsNullOrEmpty(repository?.Url) ||
102+
!Uri.TryCreate(repository.Url, UriKind.Absolute, out var repositoryUri))
103+
{
104+
return (null, null);
105+
}
106+
107+
if (repositoryUri.Scheme != Uri.UriSchemeHttps)
108+
{
109+
return (null, null);
110+
}
111+
112+
if (repository.Type.Length > 100)
113+
{
114+
throw new InvalidOperationException("Repository type must be less than or equal 100 characters");
115+
}
116+
117+
return (repositoryUri, repository.Type);
118+
}
119+
120+
private static List<PackageDependency> GetDependencies(NuspecReader nuspec)
121+
{
122+
var dependencies = new List<PackageDependency>();
123+
124+
foreach (var group in nuspec.GetDependencyGroups())
125+
{
126+
var targetFramework = group.TargetFramework.GetShortFolderName();
127+
128+
if (!group.Packages.Any())
129+
{
130+
dependencies.Add(new PackageDependency
131+
{
132+
Id = null,
133+
VersionRange = null,
134+
TargetFramework = targetFramework,
135+
});
136+
}
137+
138+
foreach (var dependency in group.Packages)
139+
{
140+
dependencies.Add(new PackageDependency
141+
{
142+
Id = dependency.Id,
143+
VersionRange = dependency.VersionRange?.ToString(),
144+
TargetFramework = targetFramework,
145+
});
146+
}
147+
}
148+
149+
return dependencies;
40150
}
41151
}
42152
}

‎src/BaGet.Core/Mirror/FileSystemPackageCacheService.cs‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.IO;
44
using System.Threading;
55
using System.Threading.Tasks;
6+
using BaGet.Core.Entities;
67
using BaGet.Core.Extensions;
78
using BaGet.Core.Services;
89
using NuGet.Configuration;
@@ -28,12 +29,35 @@ public FileSystemPackageCacheService(string storagePath) {
2829
_fsStorageProvider = new FilePackageStorageService(storagePath);
2930
}
3031

31-
public async Task AddPackageAsync(Stream stream)
32+
public async Task AddPackageAsync(Stream packageStream)
3233
{
33-
using (var archiveReader = new PackageArchiveReader(stream))
34+
Package package;
35+
Stream nuspecStream;
36+
Stream readmeStream;
37+
38+
using (var packageReader = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
3439
{
35-
await _fsStorageProvider.SavePackageStreamAsync(archiveReader, stream, CancellationToken.None);
40+
package = packageReader.GetPackageMetadata();
41+
nuspecStream = await packageReader.GetNuspecAsync(CancellationToken.None);
42+
43+
if (package.HasReadme)
44+
{
45+
readmeStream = await packageReader.GetReadmeAsync(CancellationToken.None);
46+
}
47+
else
48+
{
49+
readmeStream = null;
50+
}
3651
}
52+
53+
packageStream.Position = 0;
54+
55+
await _fsStorageProvider.SavePackageContentAsync(
56+
package,
57+
packageStream,
58+
nuspecStream,
59+
readmeStream,
60+
CancellationToken.None);
3761
}
3862

3963
public Task<bool> ExistsAsync(PackageIdentity package)

‎src/BaGet.Core/Services/FilePackageStorageService.cs‎

Lines changed: 56 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.IO;
33
using System.Threading;
44
using System.Threading.Tasks;
5+
using BaGet.Core.Entities;
56
using BaGet.Core.Extensions;
67
using NuGet.Packaging;
78
using NuGet.Packaging.Core;
@@ -23,40 +24,44 @@ public FilePackageStorageService(string storePath)
2324

2425
public string RootPath { get { return _storePath; } }
2526

26-
public async Task SavePackageStreamAsync(
27-
PackageArchiveReader package,
27+
public async Task SavePackageContentAsync(
28+
Package package,
2829
Stream packageStream,
29-
CancellationToken cancellationToken)
30+
Stream nuspecStream,
31+
Stream readmeStream,
32+
CancellationToken cancellationToken = default(CancellationToken))
3033
{
31-
var identity = await package.GetIdentityAsync(cancellationToken);
32-
var lowercasedId = identity.Id.ToLowerInvariant();
33-
var lowercasedNormalizedVersion = identity.Version.ToNormalizedString().ToLowerInvariant();
34+
package = package ?? throw new ArgumentNullException(nameof(package));
35+
packageStream = packageStream ?? throw new ArgumentNullException(nameof(packageStream));
36+
nuspecStream = nuspecStream ?? throw new ArgumentNullException(nameof(nuspecStream));
3437

35-
var packagePath = PackagePath(lowercasedId, lowercasedNormalizedVersion);
36-
var nuspecPath = NuspecPath(lowercasedId, lowercasedNormalizedVersion);
37-
var readmePath = ReadmePath(lowercasedId, lowercasedNormalizedVersion);
38+
var lowercasedId = package.Id.ToLowerInvariant();
39+
var lowercasedNormalizedVersion = package.VersionString.ToLowerInvariant();
3840

3941
EnsurePathExists(lowercasedId, lowercasedNormalizedVersion);
4042

41-
// TODO: Uploads should be idempotent. This should fail if and only if the blob
42-
// already exists but has different content.
43-
using (var fileStream = File.Open(packagePath, FileMode.CreateNew))
44-
{
45-
packageStream.Seek(0, SeekOrigin.Begin);
46-
47-
await packageStream.CopyToAsync(fileStream, DefaultCopyBufferSize, cancellationToken);
48-
}
43+
await SaveFileStreamAsync(
44+
lowercasedId,
45+
lowercasedNormalizedVersion,
46+
PackagePath,
47+
packageStream,
48+
cancellationToken);
4949

50-
using (var nuspec = await package.GetNuspecAsync(cancellationToken))
51-
using (var fileStream = File.Open(nuspecPath, FileMode.CreateNew))
52-
{
53-
await nuspec.CopyToAsync(fileStream, DefaultCopyBufferSize, cancellationToken);
54-
}
50+
await SaveFileStreamAsync(
51+
lowercasedId,
52+
lowercasedNormalizedVersion,
53+
NuspecPath,
54+
nuspecStream,
55+
cancellationToken);
5556

56-
using (var readme = package.GetReadme())
57-
using (var fileStream = File.Open(readmePath, FileMode.CreateNew))
57+
if (readmeStream != null)
5858
{
59-
await readme.CopyToAsync(fileStream, DefaultCopyBufferSize, cancellationToken);
59+
await SaveFileStreamAsync(
60+
lowercasedId,
61+
lowercasedNormalizedVersion,
62+
ReadmePath,
63+
readmeStream,
64+
cancellationToken);
6065
}
6166
}
6267

@@ -81,6 +86,23 @@ public Task<Stream> GetReadmeStreamAsync(PackageIdentity id)
8186
return Task.FromResult(readmeStream);
8287
}
8388

89+
private async Task SaveFileStreamAsync(
90+
string lowercasedId,
91+
string lowercasedNormalizedVersion,
92+
Func<string, string, string> pathFunc,
93+
Stream content,
94+
CancellationToken cancellationToken)
95+
{
96+
var path = pathFunc(lowercasedId, lowercasedNormalizedVersion);
97+
98+
// TODO: Uploads should be idempotent. This should fail if and only if the blob
99+
// already exists but has different content.
100+
using (var fileStream = File.Open(path, FileMode.CreateNew))
101+
{
102+
await content.CopyToAsync(fileStream, DefaultCopyBufferSize, cancellationToken);
103+
}
104+
}
105+
84106
public Task DeleteAsync(PackageIdentity id)
85107
{
86108
var lowercasedId = id.Id.ToLowerInvariant();
@@ -90,9 +112,15 @@ public Task DeleteAsync(PackageIdentity id)
90112
var nuspecPath = NuspecPath(lowercasedId, lowercasedNormalizedVersion);
91113
var readmePath = ReadmePath(lowercasedId, lowercasedNormalizedVersion);
92114

93-
File.Delete(packagePath);
94-
File.Delete(nuspecPath);
95-
File.Delete(readmePath);
115+
try
116+
{
117+
File.Delete(packagePath);
118+
File.Delete(nuspecPath);
119+
File.Delete(readmePath);
120+
}
121+
catch (DirectoryNotFoundException)
122+
{
123+
}
96124

97125
return Task.CompletedTask;
98126
}

‎src/BaGet.Core/Services/IPackageStorageService.cs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System.IO;
22
using System.Threading;
33
using System.Threading.Tasks;
4+
using BaGet.Core.Entities;
45
using NuGet.Packaging;
56
using NuGet.Packaging.Core;
67
using NuGet.Versioning;
@@ -19,13 +20,17 @@ public interface IPackageStorageService
1920
/// </summary>
2021
/// <param name="package">The package's metadata.</param>
2122
/// <param name="packageStream">The package's nupkg stream.</param>
23+
/// <param name="nuspecStream">The package's nuspec stream.</param>
24+
/// <param name="readmeStream">The package's readme stream, or null if none.</param>
2225
/// <param name="cancellationToken"></param>
2326
/// <returns></returns>
24-
Task SavePackageStreamAsync(
25-
PackageArchiveReader package,
27+
Task SavePackageContentAsync(
28+
Package package,
2629
Stream packageStream,
30+
Stream nuspecStream,
31+
Stream readmeStream,
2732
CancellationToken cancellationToken);
28-
33+
2934
/// <summary>
3035
/// Retrieve a package's nupkg stream.
3136
/// </summary>

0 commit comments

Comments
 (0)