diff --git a/eng/build.sh b/eng/build.sh index 53de7fda4c9..3c9404c8161 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -226,7 +226,7 @@ function Test() { projectname="${projectname%.*}" testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml" args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false" - args+=" -- xUnit.MaxParallelThreads=1" + "$DOTNET_INSTALL_DIR/dotnet" $args || exit $? } diff --git a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs index 8ab6a4ac584..ecba3a3fa85 100644 --- a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs +++ b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs @@ -14,11 +14,18 @@ open FSharp.Test.ScriptHelpers module Configuration = let supportedNames = set ["testlib.fsi";"testlib.fs";"test.mli";"test.ml";"test.fsi";"test.fs";"test2.fsi";"test2.fs";"test.fsx";"test2.fsx"] +[] +type ScriptSessionIsolation = Shared | Isolated + module ScriptRunner = open Internal.Utilities.Library - let private createEngine(args,version) = - getSessionForEval args version + let private getOrCreateEngine(args,version) sessionIsolation = + match sessionIsolation with + | ScriptSessionIsolation.Isolated -> + new FSharpScript(args, true, version) + | ScriptSessionIsolation.Shared -> + getSessionForEval args version let defaultDefines = [ @@ -27,12 +34,12 @@ module ScriptRunner = #endif ] - let runScriptFile version (cu:CompilationUnit) = + let runScriptFile version sessionIsolation (cu:CompilationUnit) = let cu = cu |> withDefines defaultDefines match cu with | FS fsSource -> use capture = new TestConsole.ExecutionCapture() - let engine = createEngine (fsSource.Options |> Array.ofList,version) + let engine = getOrCreateEngine (fsSource.Options |> Array.ofList,version) sessionIsolation let res = evalScriptFromDiskInSharedSession engine cu match res with | CompilationResult.Failure _ -> res @@ -88,7 +95,7 @@ module TestFrameworkAdapter = | LangVersion.SupportsMl -> "5.0", "--mlcompatibility" :: bonusArgs - let singleTestBuildAndRunAuxVersion (folder:string) bonusArgs mode langVersion = + let singleTestBuildAndRunAuxVersion (folder:string) bonusArgs mode langVersion sessionIsolation = let absFolder = Path.Combine(baseFolder,folder) let supportedNames, files = match mode with @@ -137,17 +144,17 @@ module TestFrameworkAdapter = cu |> withDebug |> withNoOptimize - |> ScriptRunner.runScriptFile langVersion + |> ScriptRunner.runScriptFile langVersion sessionIsolation |> shouldSucceed | FSC_OPTIMIZED -> cu |> withOptimize |> withNoDebug - |> ScriptRunner.runScriptFile langVersion + |> ScriptRunner.runScriptFile langVersion sessionIsolation |> shouldSucceed | FSI -> cu - |> ScriptRunner.runScriptFile langVersion + |> ScriptRunner.runScriptFile langVersion sessionIsolation |> shouldSucceed | COMPILED_EXE_APP -> cu @@ -161,7 +168,8 @@ module TestFrameworkAdapter = let singleTestBuildAndRunAux folder bonusArgs mode = singleTestBuildAndRunAuxVersion folder bonusArgs mode LangVersion.Latest let singleTestBuildAndRunVersion folder mode version = singleTestBuildAndRunAuxVersion folder [] mode version - let singleTestBuildAndRun folder mode = singleTestBuildAndRunVersion folder mode LangVersion.Latest + let singleTestBuildAndRun folder mode = singleTestBuildAndRunVersion folder mode LangVersion.Latest ScriptSessionIsolation.Shared + let singleTestBuildAndRunIsolated folder mode = singleTestBuildAndRunVersion folder mode LangVersion.Latest ScriptSessionIsolation.Isolated let singleVersionedNegTestAux folder bonusArgs version testName = singleTestBuildAndRunAuxVersion folder bonusArgs (NEG_TEST_BUILD testName) version diff --git a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs index d0cd80472f1..bc2e782a769 100644 --- a/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs +++ b/tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs @@ -352,13 +352,13 @@ let ``subtype-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/subtype" FSC_OPTI let ``subtype-FSI`` () = singleTestBuildAndRun "core/subtype" FSI [] -let ``syntax-FSC_DEBUG`` () = singleTestBuildAndRun "core/syntax" FSC_DEBUG +let ``syntax-FSC_DEBUG`` () = singleTestBuildAndRunIsolated "core/syntax" FSC_DEBUG [] -let ``syntax-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/syntax" FSC_OPTIMIZED +let ``syntax-FSC_OPTIMIZED`` () = singleTestBuildAndRunIsolated "core/syntax" FSC_OPTIMIZED [] -let ``syntax-FSI`` () = singleTestBuildAndRun "core/syntax" FSI +let ``syntax-FSI`` () = singleTestBuildAndRunIsolated "core/syntax" FSI [] let ``test int32-FSC_DEBUG`` () = singleTestBuildAndRun "core/int32" FSC_DEBUG @@ -453,10 +453,10 @@ let ``fsi_load-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/fsi-load" FSC_OP let ``fsi_load-FSI`` () = singleTestBuildAndRun "core/fsi-load" FSI [] -let ``reflect-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/reflect" FSC_OPTIMIZED +let ``reflect-FSC_OPTIMIZED`` () = singleTestBuildAndRunIsolated "core/reflect" FSC_OPTIMIZED [] -let ``reflect-FSI`` () = singleTestBuildAndRun "core/reflect" FSI +let ``reflect-FSI`` () = singleTestBuildAndRunIsolated "core/reflect" FSI let isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) diff --git a/tests/FSharp.Test.Utilities/Compiler.fs b/tests/FSharp.Test.Utilities/Compiler.fs index 2b3ccface37..9cda28613dd 100644 --- a/tests/FSharp.Test.Utilities/Compiler.fs +++ b/tests/FSharp.Test.Utilities/Compiler.fs @@ -1154,7 +1154,17 @@ module rec Compiler = evalFSharp fs script | _ -> failwith "Script evaluation is only supported for F#." - let getSessionForEval args version = new FSharpScript(additionalArgs=args,quiet=true,langVersion=version) + let internal sessionCache = + Collections.Concurrent.ConcurrentDictionary * LangVersion, FSharpScript>() + + let getSessionForEval args version = + let key = Set args, version + match sessionCache.TryGetValue(key) with + | true, script -> script + | _ -> + let script = new FSharpScript(additionalArgs=args,quiet=true,langVersion=version) + sessionCache.TryAdd(key, script) |> ignore + script let evalInSharedSession (script:FSharpScript) (cUnit: CompilationUnit) : CompilationResult = match cUnit with diff --git a/tests/FSharp.Test.Utilities/ScriptHelpers.fs b/tests/FSharp.Test.Utilities/ScriptHelpers.fs index aa0593fa090..f7b68ac812e 100644 --- a/tests/FSharp.Test.Utilities/ScriptHelpers.fs +++ b/tests/FSharp.Test.Utilities/ScriptHelpers.fs @@ -64,7 +64,10 @@ type FSharpScript(?additionalArgs: string[], ?quiet: bool, ?langVersion: LangVer Thread.CurrentThread.CurrentCulture <- Option.defaultValue Globalization.CultureInfo.InvariantCulture desiredCulture let cancellationToken = defaultArg cancellationToken CancellationToken.None - let ch, errors = fsi.EvalInteractionNonThrowing(code, cancellationToken) + let ch, errors = + // lock, because For memory conservation in CI FSharpScripts may be reused between tests + lock fsi <| fun () -> + fsi.EvalInteractionNonThrowing(code, cancellationToken) Thread.CurrentThread.CurrentCulture <- originalCulture diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs index 34a44df17ed..b17e1b1cd4b 100644 --- a/tests/FSharp.Test.Utilities/XunitHelpers.fs +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -12,9 +12,9 @@ open TestFramework open FSharp.Compiler.Diagnostics -open OpenTelemetry open OpenTelemetry.Resources open OpenTelemetry.Trace +open OpenTelemetry.Metrics /// Disables custom internal parallelization added with XUNIT_EXTRAS. /// Execute test cases in a class or a module one by one instead of all at once. Allow other collections to run simultaneously. @@ -130,6 +130,45 @@ type CustomTheoryTestCase = #endif + +type OpenTelemetryExport(testRunName, enable) = + // On Windows forwarding localhost to wsl2 docker container sometimes does not work. Use IP address instead. + let otlpEndpoint = Uri("http://127.0.0.1:4317") + + // Configure OpenTelemetry export. + let providers : IDisposable list = + if not enable then [] else + [ + // Configure OpenTelemetry tracing export. Traces can be viewed in Jaeger or other compatible tools. + OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(ActivityNames.FscSourceName) + .ConfigureResource(fun r -> r.AddService("F#") |> ignore) + .AddOtlpExporter(fun o -> + o.Endpoint <- otlpEndpoint + o.Protocol <- OpenTelemetry.Exporter.OtlpExportProtocol.Grpc + // Empirical values to ensure no traces are lost and no significant delay at the end of test run. + o.TimeoutMilliseconds <- 200 + o.BatchExportProcessorOptions.MaxQueueSize <- 16384 + o.BatchExportProcessorOptions.ScheduledDelayMilliseconds <- 100 + ) + .Build() + + // Configure OpenTelemetry metrics export. Metrics can be viewed in Prometheus or other compatible tools. + OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter("System.Runtime") + .ConfigureResource(fun r -> r.AddService(testRunName) |> ignore) + .AddOtlpExporter(fun e m -> + e.Endpoint <- otlpEndpoint + e.Protocol <- OpenTelemetry.Exporter.OtlpExportProtocol.Grpc + m.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds <- 1000 + ) + .Build() + ] + + interface IDisposable with + member this.Dispose() = + for p in providers do p.Dispose() + /// `XunitTestFramework` providing parallel console support and conditionally enabling optional xUnit customizations. type FSharpXunitFramework(sink: IMessageSink) = inherit XunitTestFramework(sink) @@ -145,33 +184,22 @@ type FSharpXunitFramework(sink: IMessageSink) = // We need AssemblyResolver already here, because OpenTelemetry loads some assemblies dynamically. AssemblyResolver.addResolver () #endif - - // Configure OpenTelemetry export. Traces can be viewed in Jaeger or other compatible tools. - use tracerProvider = - OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(ActivityNames.FscSourceName) - .ConfigureResource(fun r -> r.AddService("F#") |> ignore) - .AddOtlpExporter(fun o -> - // Empirical values to ensure no traces are lost and no significant delay at the end of test run. - o.TimeoutMilliseconds <- 200 - o.BatchExportProcessorOptions.MaxQueueSize <- 16384 - o.BatchExportProcessorOptions.ScheduledDelayMilliseconds <- 100 - ) - .Build() + let testRunName = $"RunTests_{assemblyName.Name} {Runtime.InteropServices.RuntimeInformation.FrameworkDescription}" + + use _ = new OpenTelemetryExport(testRunName, Environment.GetEnvironmentVariable("FSHARP_OTEL_EXPORT") <> null) + logConfig initialConfig log "Installing TestConsole redirection" TestConsole.install() begin - use _ = Activity.startNoTags $"RunTests_{assemblyName.Name} {Runtime.InteropServices.RuntimeInformation.FrameworkDescription}" + use _ = Activity.startNoTags testRunName // We can't just call base.RunTestCases here, because it's implementation is async void. use runner = new XunitTestAssemblyRunner (x.TestAssembly, testCases, x.DiagnosticMessageSink, executionMessageSink, executionOptions) runner.RunAsync().Wait() end - tracerProvider.ForceFlush() |> ignore - cleanUpTemporaryDirectoryOfThisTestRun () }