Tests: Reduce memory use in ComponentTests

eng/build.sh

@@ -226,7 +226,7 @@ function Test() {
   projectname="${projectname%.*}"
   testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
   args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
-  args+=" -- xUnit.MaxParallelThreads=1"
+
   "$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
 }
 

tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs

@@ -14,11 +14,18 @@ open FSharp.Test.ScriptHelpers
 module Configuration = 
     let supportedNames = set ["testlib.fsi";"testlib.fs";"test.mli";"test.ml";"test.fsi";"test.fs";"test2.fsi";"test2.fs";"test.fsx";"test2.fsx"]
 
+[<RequireQualifiedAccess>]
+type ScriptSessionIsolation = Shared | Isolated
+
 module ScriptRunner = 
     open Internal.Utilities.Library
 
-    let private createEngine(args,version) = 
-        getSessionForEval args version
+    let private getOrCreateEngine(args,version) sessionIsolation =
+        match sessionIsolation with
+        | ScriptSessionIsolation.Isolated ->
+            new FSharpScript(args, true, version)
+        | ScriptSessionIsolation.Shared ->
+            getSessionForEval args version
 
     let defaultDefines = 
         [ 
@@ -27,12 +34,12 @@ module ScriptRunner =
 #endif
         ]
 
-    let runScriptFile version (cu:CompilationUnit)  =
+    let runScriptFile version sessionIsolation (cu:CompilationUnit) =
         let cu  = cu |> withDefines defaultDefines
         match cu with 
         | FS fsSource ->
             use capture = new TestConsole.ExecutionCapture()
-            let engine = createEngine (fsSource.Options |> Array.ofList,version)
+            let engine = getOrCreateEngine (fsSource.Options |> Array.ofList,version) sessionIsolation
             let res = evalScriptFromDiskInSharedSession engine cu
             match res with
             | CompilationResult.Failure _ -> res
@@ -88,7 +95,7 @@ module TestFrameworkAdapter =
         | LangVersion.SupportsMl -> "5.0",  "--mlcompatibility" :: bonusArgs       
 
 
-    let singleTestBuildAndRunAuxVersion (folder:string) bonusArgs mode langVersion = 
+    let singleTestBuildAndRunAuxVersion (folder:string) bonusArgs mode langVersion sessionIsolation = 
         let absFolder = Path.Combine(baseFolder,folder)
         let supportedNames, files = 
             match mode with 
@@ -137,17 +144,17 @@ module TestFrameworkAdapter =
                 cu 
                 |> withDebug 
                 |> withNoOptimize  
-                |> ScriptRunner.runScriptFile langVersion 
+                |> ScriptRunner.runScriptFile langVersion sessionIsolation
                 |> shouldSucceed 
             | FSC_OPTIMIZED -> 
                 cu 
                 |> withOptimize 
                 |> withNoDebug 
-                |> ScriptRunner.runScriptFile langVersion 
+                |> ScriptRunner.runScriptFile langVersion sessionIsolation
                 |> shouldSucceed 
             | FSI -> 
                 cu 
-                |> ScriptRunner.runScriptFile langVersion 
+                |> ScriptRunner.runScriptFile langVersion sessionIsolation
                 |> shouldSucceed 
             | COMPILED_EXE_APP -> 
                 cu 
@@ -161,7 +168,8 @@ module TestFrameworkAdapter =
 
     let singleTestBuildAndRunAux folder bonusArgs mode = singleTestBuildAndRunAuxVersion folder bonusArgs mode LangVersion.Latest
     let singleTestBuildAndRunVersion folder mode version = singleTestBuildAndRunAuxVersion folder [] mode version
-    let singleTestBuildAndRun folder mode = singleTestBuildAndRunVersion folder mode LangVersion.Latest
+    let singleTestBuildAndRun folder mode = singleTestBuildAndRunVersion folder mode LangVersion.Latest ScriptSessionIsolation.Shared
+    let singleTestBuildAndRunIsolated folder mode = singleTestBuildAndRunVersion folder mode LangVersion.Latest ScriptSessionIsolation.Isolated
 
     let singleVersionedNegTestAux folder bonusArgs version testName =
         singleTestBuildAndRunAuxVersion folder bonusArgs  (NEG_TEST_BUILD testName) version

tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs

@@ -352,13 +352,13 @@ let ``subtype-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/subtype" FSC_OPTI
 let ``subtype-FSI`` () = singleTestBuildAndRun "core/subtype" FSI
 
 [<Fact>]
-let ``syntax-FSC_DEBUG`` () = singleTestBuildAndRun "core/syntax" FSC_DEBUG
+let ``syntax-FSC_DEBUG`` () = singleTestBuildAndRunIsolated "core/syntax" FSC_DEBUG
 
 [<Fact>]
-let ``syntax-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/syntax" FSC_OPTIMIZED
+let ``syntax-FSC_OPTIMIZED`` () = singleTestBuildAndRunIsolated "core/syntax" FSC_OPTIMIZED
 
 [<Fact>]
-let ``syntax-FSI`` () = singleTestBuildAndRun "core/syntax" FSI
+let ``syntax-FSI`` () = singleTestBuildAndRunIsolated "core/syntax" FSI
 
 [<Fact>]
 let ``test int32-FSC_DEBUG`` () = singleTestBuildAndRun "core/int32" FSC_DEBUG
@@ -453,10 +453,10 @@ let ``fsi_load-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/fsi-load" FSC_OP
 let ``fsi_load-FSI`` () = singleTestBuildAndRun "core/fsi-load" FSI
 
 [<Fact>]
-let ``reflect-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/reflect" FSC_OPTIMIZED
+let ``reflect-FSC_OPTIMIZED`` () = singleTestBuildAndRunIsolated "core/reflect" FSC_OPTIMIZED
 
 [<Fact>]
-let ``reflect-FSI`` () = singleTestBuildAndRun "core/reflect" FSI
+let ``reflect-FSI`` () = singleTestBuildAndRunIsolated "core/reflect" FSI
 
 let isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
 

tests/FSharp.Test.Utilities/Compiler.fs

@@ -1154,7 +1154,17 @@ module rec Compiler =
             evalFSharp fs script
         | _ -> failwith "Script evaluation is only supported for F#."
 
-    let getSessionForEval args version = new FSharpScript(additionalArgs=args,quiet=true,langVersion=version)
+    let internal sessionCache = 
+        Collections.Concurrent.ConcurrentDictionary<Set<string> * LangVersion, FSharpScript>()
+
+    let getSessionForEval args version =
+        let key = Set args, version
+        match sessionCache.TryGetValue(key) with
+        | true, script -> script
+        | _ -> 
+            let script = new FSharpScript(additionalArgs=args,quiet=true,langVersion=version)
+            sessionCache.TryAdd(key, script) |> ignore
+            script
 
     let evalInSharedSession (script:FSharpScript) (cUnit: CompilationUnit)  : CompilationResult =
         match cUnit with

tests/FSharp.Test.Utilities/ScriptHelpers.fs

@@ -64,7 +64,10 @@ type FSharpScript(?additionalArgs: string[], ?quiet: bool, ?langVersion: LangVer
         Thread.CurrentThread.CurrentCulture <- Option.defaultValue Globalization.CultureInfo.InvariantCulture desiredCulture
 
         let cancellationToken = defaultArg cancellationToken CancellationToken.None
-        let ch, errors = fsi.EvalInteractionNonThrowing(code, cancellationToken)
+        let ch, errors =
+            // lock, because For memory conservation in CI FSharpScripts may be reused between tests
+            lock fsi <| fun () ->
+                fsi.EvalInteractionNonThrowing(code, cancellationToken)
 
         Thread.CurrentThread.CurrentCulture <- originalCulture
 

tests/FSharp.Test.Utilities/XunitHelpers.fs

@@ -12,9 +12,9 @@ open TestFramework
 
 open FSharp.Compiler.Diagnostics
 
-open OpenTelemetry
 open OpenTelemetry.Resources
 open OpenTelemetry.Trace
+open OpenTelemetry.Metrics
 
 /// Disables custom internal parallelization added with XUNIT_EXTRAS.
 /// Execute test cases in a class or a module one by one instead of all at once. Allow other collections to run simultaneously.
@@ -130,6 +130,45 @@ type CustomTheoryTestCase =
 
 #endif
 
+
+type OpenTelemetryExport(testRunName, enable) =
+    // On Windows forwarding localhost to wsl2 docker container sometimes does not work. Use IP address instead.
+    let otlpEndpoint = Uri("http://127.0.0.1:4317")
+
+    // Configure OpenTelemetry export. 
+    let providers : IDisposable list =
+        if not enable then [] else
+            [
+            // Configure OpenTelemetry tracing export. Traces can be viewed in Jaeger or other compatible tools.
+            OpenTelemetry.Sdk.CreateTracerProviderBuilder()
+                .AddSource(ActivityNames.FscSourceName)
+                .ConfigureResource(fun r -> r.AddService("F#") |> ignore)
+                .AddOtlpExporter(fun o ->
+                    o.Endpoint <- otlpEndpoint
+                    o.Protocol <- OpenTelemetry.Exporter.OtlpExportProtocol.Grpc
+                    // Empirical values to ensure no traces are lost and no significant delay at the end of test run.
+                    o.TimeoutMilliseconds <- 200
+                    o.BatchExportProcessorOptions.MaxQueueSize <- 16384
+                    o.BatchExportProcessorOptions.ScheduledDelayMilliseconds <- 100
+                )
+                .Build()
+
+            // Configure OpenTelemetry metrics export. Metrics can be viewed in Prometheus or other compatible tools.
+            OpenTelemetry.Sdk.CreateMeterProviderBuilder()
+                .AddMeter("System.Runtime")
+                .ConfigureResource(fun r -> r.AddService(testRunName) |> ignore)
+                .AddOtlpExporter(fun e m ->
+                    e.Endpoint <- otlpEndpoint
+                    e.Protocol <- OpenTelemetry.Exporter.OtlpExportProtocol.Grpc
+                    m.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds <- 1000
+                )
+                .Build()
+            ]
+
+    interface IDisposable with
+        member this.Dispose() =
+            for p in providers do p.Dispose()
+
 /// `XunitTestFramework` providing parallel console support and conditionally enabling optional xUnit customizations.
 type FSharpXunitFramework(sink: IMessageSink) =
     inherit XunitTestFramework(sink)
@@ -145,33 +184,22 @@ type FSharpXunitFramework(sink: IMessageSink) =
                 // We need AssemblyResolver already here, because OpenTelemetry loads some assemblies dynamically.
                 AssemblyResolver.addResolver ()
             #endif
-
-                // Configure OpenTelemetry export. Traces can be viewed in Jaeger or other compatible tools.
-                use tracerProvider =
-                    OpenTelemetry.Sdk.CreateTracerProviderBuilder()
-                        .AddSource(ActivityNames.FscSourceName)
-                        .ConfigureResource(fun r -> r.AddService("F#") |> ignore)
-                        .AddOtlpExporter(fun o ->
-                            // Empirical values to ensure no traces are lost and no significant delay at the end of test run.
-                            o.TimeoutMilliseconds <- 200
-                            o.BatchExportProcessorOptions.MaxQueueSize <- 16384
-                            o.BatchExportProcessorOptions.ScheduledDelayMilliseconds <- 100
-                        )
-                        .Build()
 
+                let testRunName = $"RunTests_{assemblyName.Name} {Runtime.InteropServices.RuntimeInformation.FrameworkDescription}"
+
+                use _ = new OpenTelemetryExport(testRunName, Environment.GetEnvironmentVariable("FSHARP_OTEL_EXPORT") <> null)   
+
                 logConfig initialConfig
                 log "Installing TestConsole redirection"
                 TestConsole.install()
 
                 begin
-                    use _ = Activity.startNoTags $"RunTests_{assemblyName.Name} {Runtime.InteropServices.RuntimeInformation.FrameworkDescription}"
+                    use _ = Activity.startNoTags testRunName
                     // We can't just call base.RunTestCases here, because it's implementation is async void.
                     use runner = new XunitTestAssemblyRunner (x.TestAssembly, testCases, x.DiagnosticMessageSink, executionMessageSink, executionOptions)
                     runner.RunAsync().Wait()
                 end
 
-                tracerProvider.ForceFlush() |> ignore
-
                 cleanUpTemporaryDirectoryOfThisTestRun ()
         }