Adds tests for PUA characters

src/Common/Utilities/PathUtilities.cs

@@ -12,8 +12,6 @@ namespace Microsoft.Build.Tasks.SourceControl
 {
     internal static class PathUtilities
     {
-        internal static bool IsUnixLikePlatform => Path.DirectorySeparatorChar == '/';
-
         private static readonly char[] s_directorySeparators = { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar };
         private const string UncPrefix = @"\\";
         private const string UnixRoot = "/";
@@ -52,49 +50,5 @@ public static string EndWithSeparator(this string path)
 
         public static string EndWithSeparator(this string path, char separator)
             => path.EndsWithSeparator() ? path : path + separator;
-
-        /// <summary>
-        /// True if the path is an absolute path (rooted to drive or network share)
-        /// </summary>
-        public static bool IsAbsolute([NotNullWhen(true)] string? path)
-        {
-            if (path is not { Length: >0 })
-            {
-                return false;
-            }
-
-            if (IsUnixLikePlatform)
-            {
-                return path[0] == Path.DirectorySeparatorChar;
-            }
-
-            // "C:\"
-            if (IsDriveRootedAbsolutePath(path))
-            {
-                // Including invalid paths (e.g. "*:\")
-                return true;
-            }
-
-            // "\\machine\share"
-            // Including invalid/incomplete UNC paths (e.g. "\\goo")
-            return path.Length >= 2 &&
-                IsDirectorySeparator(path[0]) &&
-                IsDirectorySeparator(path[1]);
-        }
-
-        /// <summary>
-        /// True if the character is the platform directory separator character or the alternate directory separator.
-        /// </summary>
-        public static bool IsDirectorySeparator(char c)
-            => c == Path.DirectorySeparatorChar || c == Path.AltDirectorySeparatorChar;
-
-        /// <summary>
-        /// Returns true if given path is absolute and starts with a drive specification ("C:\").
-        /// </summary>
-        private static bool IsDriveRootedAbsolutePath(string path)
-        {
-            Debug.Assert(!IsUnixLikePlatform);
-            return path.Length >= 3 && path[1] == Path.VolumeSeparatorChar && IsDirectorySeparator(path[2]);
-        }
     }
 }

src/Common/Utilities/UriUtilities.cs

@@ -3,9 +3,8 @@
 // See the License.txt file in the project root for more information.
 
 using System;
-using System.Linq;
 using System.Diagnostics.CodeAnalysis;
-using System.IO;
+using System.Linq;
 
 namespace Microsoft.Build.Tasks.SourceControl
 {
@@ -67,91 +66,26 @@ public static bool TrySplitRelativeUrl(string relativeUrl, [NotNullWhen(true)] o
             return !parts.Any(part => part.Length == 0);
         }
 
+        /// <summary>
+        /// We can't also use <see cref="UriFormat.Unescaped"/> because it unescapes characters that need to remain escaped (e.g. '%').
+        /// <see cref="UriFormat.SafeUnescaped"/> does not unescape PUA characters (see https://github.com/dotnet/runtime/issues/89538),
+        /// but it otherwise works.
+        /// </summary>
+        private const UriFormat Format = UriFormat.SafeUnescaped;
+
         public static string GetScheme(this Uri uri)
-            => uri.GetComponents(UriComponents.Scheme, UriFormat.SafeUnescaped);
+            => uri.GetComponents(UriComponents.Scheme, Format);
 
         public static string GetHost(this Uri uri)
-            => uri.GetComponents(UriComponents.Host, UriFormat.SafeUnescaped);
+            => uri.GetComponents(UriComponents.Host, Format);
 
         public static string GetAuthority(this Uri uri)
-            => uri.GetComponents(UriComponents.Host | UriComponents.Port, UriFormat.SafeUnescaped);
+            => uri.GetComponents(UriComponents.Host | UriComponents.Port, Format);
 
         public static string GetPath(this Uri uri)
-            => uri.GetComponents(UriComponents.Path | UriComponents.KeepDelimiter, UriFormat.SafeUnescaped);
+            => uri.GetComponents(UriComponents.Path | UriComponents.KeepDelimiter, Format);
 
         public static string GetPathAndQuery(this Uri uri)
-            => uri.GetComponents(UriComponents.PathAndQuery, UriFormat.SafeUnescaped);
-
-        /// <summary>
-        /// Converts an absolute local file path or an absolute URL string to <see cref="Uri"/>.
-        /// </summary>
-        /// <exception cref="UriFormatException">
-        /// The <paramref name="absolutePath"/> can't be represented as <see cref="Uri"/>.
-        /// For example, UNC paths with invalid characters in server name.
-        /// </exception>
-        public static Uri CreateAbsoluteUri(string absolutePath)
-        {
-            var uriString = IsAscii(absolutePath) ? absolutePath : GetAbsoluteUriString(absolutePath);
-            try
-            {
-#pragma warning disable RS0030 // Do not use banned APIs
-                return new(uriString, UriKind.Absolute);
-#pragma warning restore
-
-            }
-            catch (UriFormatException e)
-            {
-                // The standard URI format exception does not include the failing path, however
-                // in pretty much all cases we need to know the URI string (and original string) in order to fix the issue.
-                throw new UriFormatException($"Failed create URI from '{uriString}'; original string: '{absolutePath}'", e);
-            }
-        }
-
-        // Implements workaround for https://github.com/dotnet/runtime/issues/89538:
-        internal static string GetAbsoluteUriString(string absolutePath)
-        {
-            if (!PathUtilities.IsAbsolute(absolutePath))
-            {
-                return absolutePath;
-            }
-
-            var parts = absolutePath.Split([Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar]);
-
-            if (PathUtilities.IsUnixLikePlatform)
-            {
-                // Unix path: first part is empty, all parts should be escaped
-                return "file://" + string.Join("/", parts.Select(EscapeUriPart));
-            }
-
-            if (parts is ["", "", var serverName, ..])
-            {
-                // UNC path: first non-empty part is server name and shouldn't be escaped
-                return "file://" + serverName + "/" + string.Join("/", parts.Skip(3).Select(EscapeUriPart));
-            }
-
-            // Drive-rooted path: first part is "C:" and shouldn't be escaped
-            return "file:///" + parts[0] + "/" + string.Join("/", parts.Skip(1).Select(EscapeUriPart));
-
-#pragma warning disable SYSLIB0013 // Type or member is obsolete
-            static string EscapeUriPart(string stringToEscape)
-                => Uri.EscapeUriString(stringToEscape).Replace("#", "%23");
-#pragma warning restore
-        }
-
-        private static bool IsAscii(char c)
-            => (uint)c <= '\x007f';
-
-        private static bool IsAscii(string filePath)
-        {
-            for (var i = 0; i < filePath.Length; i++)
-            {
-                if (!IsAscii(filePath[i]))
-                {
-                    return false;
-                }
-            }
-
-            return true;
-        }
+            => uri.GetComponents(UriComponents.PathAndQuery, Format);
     }
 }

src/Microsoft.Build.Tasks.Git.UnitTests/GitIgnoreTests.cs

@@ -46,6 +46,7 @@ internal void TryParsePattern(string line, string glob, GitIgnore.PatternFlags f
         [InlineData("//")]
         [InlineData("!/")]
         [InlineData("!//")]
+        [InlineData("#" + TestStrings.GB18030)]
         public void TryParsePattern_None(string line)
         {
             Assert.False(GitIgnore.TryParsePattern(line, new StringBuilder(), out _, out _));
@@ -69,16 +70,16 @@ public void IsIgnored_CaseSensitive()
             var dirC = dirB.CreateDirectory("C");
             dirC.CreateDirectory("D1");
             dirC.CreateDirectory("D2");
-            dirC.CreateDirectory("D3");
+            dirC.CreateDirectory(TestStrings.GB18030);
 
-            dirA.CreateFile(".gitignore").WriteAllText(@"
+            dirA.CreateFile(".gitignore").WriteAllText($@"
 !z.txt
 *.txt
 !u.txt
 !v.txt
 !.git
 b/
-D3/
+{TestStrings.GB18030}/
 Bar/**/*.xyz
 v.txt
 ");
@@ -129,7 +130,7 @@ public void IsIgnored_CaseSensitive()
             Assert.True(matcher.IsPathIgnored(Path.Combine(workingDir.Path, "A", "B", "C", "D1", "b") + Path.DirectorySeparatorChar));
 
             // matches "D3/" (existing directory path)
-            Assert.True(matcher.IsPathIgnored(Path.Combine(workingDir.Path, "A", "B", "C", "D3")));
+            Assert.True(matcher.IsPathIgnored(Path.Combine(workingDir.Path, "A", "B", "C", TestStrings.GB18030)));
 
             // matches "D1/c.cs"
             Assert.True(matcher.IsPathIgnored(Path.Combine(workingDir.Path, "A", "B", "C", "D1", "c.cs")));
@@ -150,7 +151,7 @@ public void IsIgnored_CaseSensitive()
                 "/Repo/A/B/C/D1: False",
                 "/Repo/A/B/C/D2/E: True",
                 "/Repo/A/B/C/D2: True",
-                "/Repo/A/B/C/D3: True",
+                $"/Repo/A/B/C/{TestStrings.GB18030}: True",
                 "/Repo/A/B/C: False",
                 "/Repo/A/B: False",
                 "/Repo/A: False",

src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs

@@ -201,7 +201,7 @@ public void GetRepositoryUrl_UnsupportedUrl(string kind)
             using var temp = new TempRoot();
 
             var dir = temp.CreateDirectory();
-            var originRepoPath = dir.CreateDirectory("x \u1234").Path;
+            var originRepoPath = dir.CreateDirectory("x " + TestStrings.GB18030).Path;
 
             var url = kind switch
             {
@@ -229,50 +229,19 @@ public void GetRepositoryUrl_UnsupportedUrl(string kind)
         [InlineData("abc://user@github.com/org/repo")]
         public void NormalizeUrl_PlatformAgnostic1(string url)
         {
-            Assert.Equal(url, GitOperations.NormalizeUrl(url, s_root)?.AbsoluteUri);
+            AssertEx.AreEqual(url, GitOperations.NormalizeUrl(url, s_root)?.AbsoluteUri);
         }
 
         [Theory]
         [InlineData("http://?", null)]
         [InlineData("https://github.com/org/repo/./.", "https://github.com/org/repo/")]
-        [InlineData("http://github.com/org/\u1234", "http://github.com/org/%E1%88%B4")]
+        [InlineData("http://github.com/org/" + TestStrings.RepoName, "http://github.com/org/" + TestStrings.RepoNameFullyEscaped)]
         [InlineData("ssh://github.com/org/../repo", "ssh://github.com/repo")]
         [InlineData("ssh://github.com/%32/repo", "ssh://github.com/2/repo")]
         [InlineData("ssh://github.com/%3F/repo", "ssh://github.com/%3F/repo")]
         public void NormalizeUrl_PlatformAgnostic2(string url, string expectedUrl)
         {
-            Assert.Equal(expectedUrl, GitOperations.NormalizeUrl(url, s_root)?.AbsoluteUri);
-        }
-
-        [ConditionalTheory(typeof(WindowsOnly))]
-        [InlineData(@"C:", "file:///C:/")]
-        [InlineData(@"C:\", "file:///C:/")]
-        [InlineData(@"C:x", null)]
-        [InlineData(@"C:x\y\..\z", null)]
-        [InlineData(@"C:org/repo", null)]
-        [InlineData(@"D:\src", "file:///D:/src")]
-        [InlineData(@"D:\a%20b", "file:///D:/a%2520b")]
-        [InlineData(@"\\", null)]
-        [InlineData(@"\\server", "file://server/")]
-        [InlineData(@"\\server\dir", "file://server/dir")]
-        [InlineData(@"relative/./path", "file:///C:/src/a/b/relative/path")]
-        [InlineData(@"%20", "file:///C:/src/a/b/%2520")]
-        [InlineData(@"..\%20", "file:///C:/src/a/%2520")]
-        [InlineData(@"../relative/path", "file:///C:/src/a/relative/path")]
-        [InlineData(@"..\relative\path", "file:///C:/src/a/relative/path")]
-        [InlineData(@"../relative/path?a=b", "file:///C:/src/a/relative/path%3Fa=b")]
-        [InlineData(@"../relative/path*<>|\0%00", "file:///C:/src/a/relative/path*%3C%3E%7C/0%2500")]
-        [InlineData(@"../../../../relative/path", "file:///C:/relative/path")]
-        [InlineData(@"a:/../../relative/path", "file:///a:/relative/path")]
-        [InlineData(@"Z:/a/b/../../relative/path", "file:///Z:/relative/path")]
-        [InlineData(@"../.://../../relative/path", "file:///C:/src/a/relative/path")]
-        [InlineData(@"../.:./../../relative/path", "ssh://../relative/path")]
-        [InlineData(@".:/../../relative/path", "ssh://./relative/path")]
-        [InlineData(@"..:/../../relative/path", "ssh://../relative/path")]
-        [InlineData(@"@:org/repo", "file:///C:/src/a/b/@:org/repo")]
-        public void NormalizeUrl_Windows(string url, string expectedUrl)
-        {
-            Assert.Equal(expectedUrl, GitOperations.NormalizeUrl(url, @"C:\src\a\b")?.AbsoluteUri);
+            AssertEx.AreEqual(expectedUrl, GitOperations.NormalizeUrl(url, s_root)?.AbsoluteUri);
         }
 
         [ConditionalTheory(typeof(UnixOnly))]
@@ -461,8 +430,7 @@ public void GetSourceRoots_RelativeSubmodulePath(bool warnOnMissingCommitOrUnsup
 
             var dir = temp.CreateDirectory();
 
-            // TODO: test unicode chars on Linux as well https://github.com/dotnet/corefx/issues/34227
-            var repoDir = dir.CreateDirectory("%25@" + (s == '\\' ? "噸" : ""));
+            var repoDir = dir.CreateDirectory("%25@噸" + TestStrings.GB18030);
 
             var repo1WorkingDir = dir.CreateDirectory("1");
             var repo1GitDir = repo1WorkingDir.CreateDirectory(".git");

src/Microsoft.Build.Tasks.Git/GitOperations.cs

@@ -26,7 +26,7 @@ internal static class GitOperations
         public static string? GetRepositoryUrl(GitRepository repository, string? remoteName, bool warnOnMissingOrUnsupportedRemote = true, Action<string, object?[]>? logWarning = null)
         {
             NullableDebug.Assert(repository.WorkingDirectory != null);
-            
+
             var remoteUrl = GetRemoteUrl(repository, ref remoteName, warnOnMissingOrUnsupportedRemote, logWarning);
             if (remoteUrl == null)
             {
@@ -181,6 +181,10 @@ private static bool IsSupportedScheme(string scheme)
                 return uri;
             }
 
+            // Note that creating a Uri from a local path with certain Unicode characters
+            // has issues (https://github.com/dotnet/runtime/issues/89538). 
+            // However, we do not support file:// URIs, so if the actual URI
+            // value doesn't matter for local paths.
             if (!Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out uri))
             {
                 return null;

src/Microsoft.Build.Tasks.Git/LocateRepository.cs

@@ -50,7 +50,7 @@ public sealed class LocateRepository : RepositoryTask
         private protected override void Execute(GitRepository repository)
         {
             NullableDebug.Assert(repository.WorkingDirectory != null);
-            
+
             RepositoryId = repository.GitDirectory;
             WorkingDirectory = repository.WorkingDirectory;
             Url = GitOperations.GetRepositoryUrl(repository, RemoteName, warnOnMissingOrUnsupportedRemote: !NoWarnOnMissingInfo, Log.LogWarning);

src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs

@@ -89,7 +89,7 @@ public void BuildSourceLinkUrl_BitbucketEnterprise_PersonalToken()
             var task = new GetSourceLinkUrl()
             {
                 BuildEngine = engine,
-                SourceRoot = new MockItem("/src/", KVP("RepositoryUrl", ProjectCollection.Escape("https://user_name%40domain.com:Bitbucket_personaltoken@bitbucket.domain.tools/scm/abc/project1.git")), KVP("SourceControl", "git"), KVP("RevisionId", "0123456789abcdefABCDEF000000000000000000")),
+                SourceRoot = new MockItem("/src/", KVP("RepositoryUrl", "https://user_name%40domain.com:Bitbucket_personaltoken@bitbucket.domain.tools/scm/abc/project1.git"), KVP("SourceControl", "git"), KVP("RevisionId", "0123456789abcdefABCDEF000000000000000000")),
                 Hosts = new[]
                 {
                     new MockItem("bitbucket.domain.tools", KVP("ContentUrl", "https://bitbucket.domain.tools")),