Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/Cli/dotnet/Parser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public static Command GetBuiltInCommand(string commandName)
// all of the parameters of their wrapped command by design)
// error. so `dotnet msbuild /t:thing` throws a parse error.
// .UseParseErrorReporting(127)
.UseParseErrorReporting("new")
.UseParseErrorReporting("new", "test")
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need test to be part of the new error reporting as well if we want the nice parser errors to be shown to the user. now that we do explicit forwarding of args to commands like MSbuild, we could potentially remove this in favor of the standard UseParseErrorReporting() middleware, but I didn't think that was necessary for your ---removal PR at this time. It can come next :)

.UseHelp()
.UseHelpBuilder(context => DotnetHelpBuilder.Instance.Value)
.UseLocalizationResources(new CommandLineValidationMessages())
Expand All @@ -127,12 +127,12 @@ public static Command GetBuiltInCommand(string commandName)
.DisablePosixBinding()
.Build();

private static CommandLineBuilder UseParseErrorReporting(this CommandLineBuilder builder, string commandName)
private static CommandLineBuilder UseParseErrorReporting(this CommandLineBuilder builder, params string[] commandName)
{
builder.AddMiddleware(async (context, next) =>
{
CommandResult currentCommandResult = context.ParseResult.CommandResult;
while (currentCommandResult != null && currentCommandResult.Command.Name != commandName)
while (currentCommandResult != null && !(commandName.Contains(currentCommandResult.Command.Name)))
{
currentCommandResult = currentCommandResult.Parent as CommandResult;
}
Expand Down
15 changes: 15 additions & 0 deletions src/Cli/dotnet/commands/dotnet-test/LocalizableStrings.resx
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,21 @@
<data name="AppDescription" xml:space="preserve">
<value>Test Driver for the .NET Platform</value>
</data>
<data name="AdditionalArguments" xml:space="preserve">
<value>forwarded arguments</value>
</data>
<data name="AdditionalArgumentsDescription" xml:space="preserve">
<value>Specifies extra arguments to pass to the adapter. Use a space to separate multiple arguments. If the item being tested is a project, solution, or directory this argument will be passed to MSBuild. If the item being tested is a .dll or an .exe this argument will be passed to vstest.</value>
</data>
<data name="CouldNotParseRunSetting" xml:space="preserve">
<value>Argument '{0}' could not be parsed as a RunSetting. Use a key/value pair separated with an equals character, like 'foo=bar'</value>
</data>
<data name="RunSettings" xml:space="preserve">
<value>run settings</value>
</data>
<data name="RunSettingsDescription" xml:space="preserve">
<value>KEY=VALUE pairs that replace or augment RunSettings values from any specified .runsettings files </value>
</data>
<data name="CmdSettingsFile" xml:space="preserve">
<value>SETTINGS_FILE</value>
</data>
Expand Down
49 changes: 24 additions & 25 deletions src/Cli/dotnet/commands/dotnet-test/Program.cs
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
// Copyright (c) .NET Foundation and contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

#nullable enable

using System;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.Parsing;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.DotNet.Cli;
using Microsoft.DotNet.Cli.Utils;

Expand All @@ -17,7 +20,7 @@ public class TestCommand : RestoringCommand
public TestCommand(
IEnumerable<string> msbuildArgs,
bool noRestore,
string msbuildPath = null)
string? msbuildPath = null)
: base(msbuildArgs, noRestore, msbuildPath)
{
}
Expand All @@ -32,23 +35,19 @@ public static int Run(ParseResult parseResult)
// from the VSTest side.
string testSessionCorrelationId = $"{Environment.ProcessId}_{Guid.NewGuid()}";

string[] args = parseResult.GetArguments();
var args = parseResult.GetValueForArgument(TestCommandParser.ForwardedArgs) ?? Array.Empty<string>();
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this logic is a lot more understandable - let me know what you think

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Really better.
But I think that we can remove ?? Array.Empty<string>() part

var settings = parseResult.GetValueForArgument(TestCommandParser.InlineRunSettings) ?? Array.Empty<(string key, string value)>();

if (VSTestTrace.TraceEnabled)
{
string commandLineParameters = "";
if (args?.Length > 0)
if (args.Length > 0)
{
commandLineParameters = args.Aggregate((a, b) => $"{a} | {b}");
}
VSTestTrace.SafeWriteTrace(() => $"Argument list: '{commandLineParameters}'");
}

// settings parameters are after -- (including --), these should not be considered by the parser
string[] settings = args.SkipWhile(a => a != "--").ToArray();
// all parameters before --
args = args.TakeWhile(a => a != "--").ToArray();

// Fix for https://github.com/Microsoft/vstest/issues/1453
// Run dll/exe directly using the VSTestForwardingApp
if (ContainsBuiltTestSources(args))
Expand All @@ -59,11 +58,11 @@ public static int Run(ParseResult parseResult)
return ForwardToMsbuild(parseResult, settings, testSessionCorrelationId);
}

private static int ForwardToMsbuild(ParseResult parseResult, string[] settings, string testSessionCorrelationId)
private static int ForwardToMsbuild(ParseResult parseResult, (string key, string value)[] settings, string testSessionCorrelationId)
{
// Workaround for https://github.com/Microsoft/vstest/issues/1503
const string NodeWindowEnvironmentName = "MSBUILDENSURESTDOUTFORTASKPROCESSES";
string previousNodeWindowSetting = Environment.GetEnvironmentVariable(NodeWindowEnvironmentName);
string? previousNodeWindowSetting = Environment.GetEnvironmentVariable(NodeWindowEnvironmentName);
try
{
Environment.SetEnvironmentVariable(NodeWindowEnvironmentName, "1");
Expand All @@ -80,7 +79,10 @@ private static int ForwardToMsbuild(ParseResult parseResult, string[] settings,
}
}

private static int ForwardToVSTestConsole(ParseResult parseResult, string[] args, string[] settings, string testSessionCorrelationId)
private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\"");
private static string FormatRunSetting((string key, string value) kv) => $"{Escape(kv.key)}={Escape(kv.value)}";

private static int ForwardToVSTestConsole(ParseResult parseResult, string[] args, (string key, string value)[] settings, string testSessionCorrelationId)
{
List<string> convertedArgs = new VSTestArgumentConverter().Convert(args, out List<string> ignoredArgs);
if (ignoredArgs.Any())
Expand All @@ -90,7 +92,7 @@ private static int ForwardToVSTestConsole(ParseResult parseResult, string[] args

// merge the args settings, we don't need to escape
// one more time, there is no extra hop via msbuild
convertedArgs.AddRange(settings);
convertedArgs.AddRange(settings.Select(FormatRunSetting));

if (!FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_ARTIFACTS_POSTPROCESSING))
{
Expand All @@ -107,7 +109,7 @@ private static int ForwardToVSTestConsole(ParseResult parseResult, string[] args
return exitCode;
}

private static TestCommand FromParseResult(ParseResult result, string[] settings, string testSessionCorrelationId, string msbuildPath = null)
private static TestCommand FromParseResult(ParseResult result, (string key, string value)[] settings, string testSessionCorrelationId, string? msbuildPath = null)
{
result.ShowHelpOrErrorIfAppropriate();

Expand All @@ -122,27 +124,24 @@ private static TestCommand FromParseResult(ParseResult result, string[] settings

if (settings.Any())
{
//workaround for correct -- logic
var commandArgument = result.GetValueForArgument(TestCommandParser.SlnOrProjectArgument);
if(!string.IsNullOrWhiteSpace(commandArgument) && !settings.Contains(commandArgument))

var builder = new StringBuilder();
foreach (var kv in settings)
{
msbuildArgs.Add(result.GetValueForArgument(TestCommandParser.SlnOrProjectArgument));
builder.Append(FormatRunSetting(kv));
builder.Append(';');
}

// skip '--' and escape every \ to be \\ and every " to be \" to survive the next hop
string[] escaped = settings.Skip(1).Select(s => s.Replace("\\", "\\\\").Replace("\"", "\\\"")).ToArray();

string runSettingsArg = string.Join(";", escaped);
msbuildArgs.Add($"-property:VSTestCLIRunSettings=\"{runSettingsArg}\"");
msbuildArgs.Add($"-property:VSTestCLIRunSettings=\"{builder.ToString()}\"");
}
else
{
var argument = result.GetValueForArgument(TestCommandParser.SlnOrProjectArgument);
if(!string.IsNullOrWhiteSpace(argument))
if (!string.IsNullOrWhiteSpace(argument))
msbuildArgs.Add(argument);
}

string verbosityArg = result.ForwardedOptionValues<IReadOnlyCollection<string>>(TestCommandParser.GetCommand(), "verbosity")?.SingleOrDefault() ?? null;
string? verbosityArg = result.ForwardedOptionValues<IReadOnlyCollection<string>>(TestCommandParser.GetCommand(), "verbosity")?.SingleOrDefault() ?? null;
if (verbosityArg != null)
{
string[] verbosity = verbosityArg.Split(':', 2);
Expand Down Expand Up @@ -249,7 +248,7 @@ private static void SetEnvironmentVariablesFromParameters(TestCommand testComman
return;
}

foreach (string env in parseResult.GetValueForOption(option))
foreach (string env in parseResult.GetValueForOption(option)!)
{
string name = env;
string value = string.Empty;
Expand Down
125 changes: 124 additions & 1 deletion src/Cli/dotnet/commands/dotnet-test/TestCommandParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.CommandLine;
using System.CommandLine.Invocation;
using System.CommandLine.Parsing;
using System.IO;
using System.Linq;
using Microsoft.DotNet.Tools;
using Microsoft.DotNet.Tools.Test;
Expand All @@ -17,12 +18,132 @@ internal static class TestCommandParser
{
public static readonly string DocsLink = "https://aka.ms/dotnet-test";

public static readonly Argument<string> SlnOrProjectArgument = new Argument<string>(CommonLocalizableStrings.SolutionOrProjectArgumentName)
/// <summary>
/// Parser delegate that only accepts a token that is a
/// * project or solution file
/// * directory that contains a project or solution
/// * .dll or .exe file
/// </summary>
/// <remarks>
/// S.CL usage note - OnlyTake(0) signals that this token should be returned to the
/// token stream for the next argument. In this way we prevent initial tokens that
/// are not relevant from being captured in this argument, since the syntax is a bit
/// ambiguous.
/// </remarks>
private static ParseArgument<string> IsProjectOrSln =
(ctx) =>
{
bool HasProjectOrSolution(DirectoryInfo dir) =>
dir.EnumerateFiles("*.*proj").Any() || dir.EnumerateFiles(".sln").Any();

if (ctx.Tokens.Count == 0)
{
ctx.OnlyTake(0);
return null;
}
else
{
var tokenValue = ctx.Tokens[0].Value;
var ext = System.IO.Path.GetExtension(tokenValue);
if (ext.EndsWith("proj") || ext.EndsWith(".sln") || ext.EndsWith(".dll") || ext.EndsWith(".exe"))
{
ctx.OnlyTake(1);
return tokenValue;
}
else
{
var path = System.IO.Path.GetFullPath(tokenValue);
var dir = new System.IO.DirectoryInfo(path);
if (dir.Exists && HasProjectOrSolution(dir))
{
ctx.OnlyTake(1);
return tokenValue;
}

ctx.OnlyTake(0);
return null;
}
}
};

public static readonly Argument<string> SlnOrProjectArgument = new Argument<string>(CommonLocalizableStrings.SolutionOrProjectArgumentName, parse: IsProjectOrSln)
{
Description = CommonLocalizableStrings.SolutionOrProjectArgumentDescription,
Arity = ArgumentArity.ZeroOrOne
};

public static (bool success, string key, string value) TryParseRunSetting (string token) {
var parts = token.Split('=');
if (parts.Length == 2) {
return (true, parts[0], parts[1]);
}
return (false, null, null);
}

/// <summary>
/// A parser that takes from the start of the token stream until the the first argument that could be a RunSetting
/// </summary>
private static ParseArgument<string[]> ParseUntilFirstPotentialRunSetting = ctx =>
{
if (ctx.Tokens.Count == 0)
{
ctx.OnlyTake(0);
return Array.Empty<string>();
}
var tokens = new List<string>();
foreach (var token in ctx.Tokens) {
var (success, key, value) = TryParseRunSetting(token.Value);
if (success) {
break;
}
else
{
tokens.Add(token.Value);
}
}
ctx.OnlyTake(tokens.Count);
return tokens.ToArray();
};

public static readonly Argument<string[]> ForwardedArgs = new(LocalizableStrings.AdditionalArguments, parse: ParseUntilFirstPotentialRunSetting, description: LocalizableStrings.AdditionalArgumentsDescription)
{
Arity = ArgumentArity.ZeroOrMore
};

private static ParseArgument<(string key, string value)[]> ParseRunSettings = ctx =>
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it ok that we don't check double dash? Seems like we can pass dotnet test --logger "trx;logfilename=custom.trx" RunConfiguration.ResultDirectory=

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can actually test this now - I should add some tests where we take certain suspicious command lines, run them through the parser, and then do assertions on the parsed arguments. Do you have any specific command lines you'd recommend testing against?

Copy link
Owner

@Tunduk Tunduk Jun 23, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to run GivenDotnetTestBuildsAndRunsTestFromCsproj.ItAcceptsMultipleLoggersAsCliArguments test and it failed. This test was my pain and I remembered it =)
I think problem that System.CommandLine wasn't deployed with dotnet/command-line-api#1759 fix. And I can't find new version in nuget.

Unfortunately I don't use dotnet test very often, so nothing comes to mind =(
But I think that there are only 3 combinations that we should check.

Copy link
Owner

@Tunduk Tunduk Jun 23, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My bad. System.CommandLine was updated.

Copy link
Owner

@Tunduk Tunduk Jun 23, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I reproduced problem. Something odd happens when we pass 3 arguments to command and call command with 1 token.
If we call with >1 token then we will get exception from System.CommandLine.
Like this.

I'll create an issue for System.CommandLine team tomorrow
test.txt

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

{
if (ctx.Tokens.Count == 0)
{
ctx.OnlyTake(0);
return Array.Empty<(string key, string value)>();
}
var settings = new List<(string key, string value)>(ctx.Tokens.Count);
var consumed = 0;
foreach (var token in ctx.Tokens)
{
var parts = token.Value.Split('=', 2);
if (parts.Length == 2)
{
consumed += 1;
settings.Add((parts[0], parts[1]));
}
else
{
//$"Argument '{token.Value}' could not be parsed as a RunSetting. Use a key/value pair separated with an equals character, like 'foo=bar'";
ctx.ErrorMessage = String.Format(LocalizableStrings.CouldNotParseRunSetting, token.Value);
ctx.OnlyTake(consumed);
return null;
}
}
ctx.OnlyTake(consumed);
return settings.ToArray();
};

public static readonly Argument<(string key, string value)[]> InlineRunSettings = new(LocalizableStrings.RunSettings, parse: ParseRunSettings, description: LocalizableStrings.RunSettingsDescription)
{
Arity = ArgumentArity.ZeroOrMore
};

public static readonly Option<string> SettingsOption = new ForwardedOption<string>(new string[] { "-s", "--settings" }, LocalizableStrings.CmdSettingsDescription)
{
ArgumentHelpName = LocalizableStrings.CmdSettingsFile
Expand Down Expand Up @@ -159,6 +280,8 @@ private static Command ConstructCommand()
command.AddOption(CommonOptions.VerbosityOption);
command.AddOption(CommonOptions.ArchitectureOption);
command.AddOption(CommonOptions.OperatingSystemOption);
command.AddArgument(ForwardedArgs);
command.AddArgument(InlineRunSettings);

command.SetHandler(TestCommand.Run);

Expand Down
25 changes: 25 additions & 0 deletions src/Cli/dotnet/commands/dotnet-test/xlf/LocalizableStrings.cs.xlf
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 xliff-core-1.2-transitional.xsd">
<file datatype="xml" source-language="en" target-language="cs" original="../LocalizableStrings.resx">
<body>
<trans-unit id="AdditionalArguments">
<source>forwarded arguments</source>
<target state="new">forwarded arguments</target>
<note />
</trans-unit>
<trans-unit id="AdditionalArgumentsDescription">
<source>Specifies extra arguments to pass to the adapter. Use a space to separate multiple arguments. If the item being tested is a project, solution, or directory this argument will be passed to MSBuild. If the item being tested is a .dll or an .exe this argument will be passed to vstest.</source>
<target state="new">Specifies extra arguments to pass to the adapter. Use a space to separate multiple arguments. If the item being tested is a project, solution, or directory this argument will be passed to MSBuild. If the item being tested is a .dll or an .exe this argument will be passed to vstest.</target>
<note />
</trans-unit>
<trans-unit id="AppFullName">
<source>.NET Test Driver</source>
<target state="translated">Testovací ovladač .NET</target>
Expand Down Expand Up @@ -195,6 +205,11 @@ Pokud zadaný adresář neexistuje, bude vytvořen.</target>
<target state="translated">RESULTS_DIR</target>
<note />
</trans-unit>
<trans-unit id="CouldNotParseRunSetting">
<source>Argument '{0}' could not be parsed as a RunSetting. Use a key/value pair separated with an equals character, like 'foo=bar'</source>
<target state="new">Argument '{0}' could not be parsed as a RunSetting. Use a key/value pair separated with an equals character, like 'foo=bar'</target>
<note />
</trans-unit>
<trans-unit id="CrashDumpTypeArgumentName">
<source>DUMP_TYPE</source>
<target state="translated">DUMP_TYPE</target>
Expand All @@ -215,6 +230,11 @@ Pokud zadaný adresář neexistuje, bude vytvořen.</target>
<target state="translated">Následující argumenty se ignorovaly: {0}</target>
<note />
</trans-unit>
<trans-unit id="RunSettings">
<source>run settings</source>
<target state="new">run settings</target>
<note />
</trans-unit>
<trans-unit id="RunSettingsArgumentsDescription">
<source>

Expand All @@ -232,6 +252,11 @@ Argumenty RunSettings:
Příklad: dotnet test -- MSTest.DeploymentEnabled=false MSTest.MapInconclusiveToFailed=True</target>
<note />
</trans-unit>
<trans-unit id="RunSettingsDescription">
<source>KEY=VALUE pairs that replace or augment RunSettings values from any specified .runsettings files </source>
<target state="new">KEY=VALUE pairs that replace or augment RunSettings values from any specified .runsettings files </target>
<note />
</trans-unit>
<trans-unit id="cmdCollectFriendlyName">
<source>DATA_COLLECTOR_NAME</source>
<target state="translated">DATA_COLLECTOR_NAME</target>
Expand Down
Loading