fix tests, update baselines and split into stdout/stderr

VisualFSharp.sln

@@ -28,6 +28,9 @@ EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler", "src\fsharp\FSharp.Compiler\FSharp.Compiler.fsproj", "{2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}"
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.VS.FSI", "vsintegration\src\vs\FsPkgs\FSharp.VS.FSI\FSharp.VS.FSI.fsproj", "{991DCF75-C2EB-42B6-9A0D-AA1D2409D519}"
+	ProjectSection(ProjectDependencies) = postProject
+		{2E4D67B4-522D-4CF7-97E4-BA940F0B18F3} = {2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}
+	EndProjectSection
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler.Server.Shared", "src\fsharp\FSharp.Compiler.Server.Shared\FSharp.Compiler.Server.Shared.fsproj", "{D5870CF0-ED51-4CBC-B3D7-6F56DA84AC06}"
 EndProject

appveyor.yml

@@ -2,7 +2,8 @@ os: Visual Studio 2015
 
 environment:
   matrix:
-    - BUILD_PROFILE: ci
+    - BUILD_PROFILE: ci_part1
+    - BUILD_PROFILE: ci_part2
 
 init: 
 build_script: 

build.cmd

@@ -13,7 +13,7 @@ echo Build and run a subset of test suites
 echo.
 echo Usage:
 echo.
-echo build.cmd ^<all^|build^|debug^|release^|compiler^|pcls^|vs^|ci^|ci_part1^|ci_part2^|ci_part3^>
+echo build.cmd ^<all^|build^|debug^|release^|compiler^|pcls^|vs^|ci^|ci_part1^|ci_part2^>
 echo.
 echo No arguments default to 'build' 
 echo.
@@ -28,17 +28,12 @@ exit /b 1
 
 set BUILD_PROTO=0
 set BUILD_NET40=1
-set BUILD_PORTABLE47=0
-set BUILD_PORTABLE7=0
-set BUILD_PORTABLE78=0
-set BUILD_PORTABLE259=0
+set BUILD_PORTABLE=0
 set BUILD_VS=0
+set BUILD_FSHARP_DATA_TYPEPROVIDERS=0
 set TEST_COMPILERUNIT=0
 set TEST_NET40_COREUNIT=0
-set TEST_PORTABLE47_COREUNIT=0
-set TEST_PORTABLE7_COREUNIT=0
-set TEST_PORTABLE78_COREUNIT=0
-set TEST_PORTABLE259_COREUNIT=0
+set TEST_PORTABLE_COREUNIT=0
 set TEST_VS=0
 set TEST_FSHARP_SUITE=0
 set TEST_TAGS=
@@ -74,14 +69,8 @@ if /i '%ARG%' == 'compiler' (
 )
 
 if /i '%ARG%' == 'pcls' (
-    set BUILD_PORTABLE47=1
-    set BUILD_PORTABLE7=1
-    set BUILD_PORTABLE78=1
-    set BUILD_PORTABLE259=1
-    set TEST_PORTABLE47_COREUNIT=1
-    set TEST_PORTABLE7_COREUNIT=1
-    set TEST_PORTABLE78_COREUNIT=1
-    set TEST_PORTABLE259_COREUNIT=1
+    set BUILD_PORTABLE=1
+    set TEST_PORTABLE_COREUNIT=1
 )
 
 if /i '%ARG%' == 'vs' (
@@ -91,34 +80,24 @@ if /i '%ARG%' == 'vs' (
 
 if /i '%ARG%' == 'all' (
     set BUILD_PROTO=1
-    set BUILD_PORTABLE47=1
-    set BUILD_PORTABLE7=1
-    set BUILD_PORTABLE78=1
-    set BUILD_PORTABLE259=1
+    set BUILD_PORTABLE=1
     set BUILD_VS=1
+    set BUILD_FSHARP_DATA_TYPEPROVIDERS=1
     set TEST_COMPILERUNIT=1
-    set TEST_PORTABLE47_COREUNIT=1
-    set TEST_PORTABLE7_COREUNIT=1
-    set TEST_PORTABLE78_COREUNIT=1
-    set TEST_PORTABLE259_COREUNIT=1
+    set TEST_PORTABLE_COREUNIT=1
     set TEST_VS=1
     set TEST_FSHARP_SUITE=1
     set TEST_FSHARPQA_SUITE=1
 )
 
 REM Same as 'all' but smoke testing only
 if /i '%ARG%' == 'ci' (
-    set BUILD_PORTABLE47=1
-    set BUILD_PORTABLE7=1
-    set BUILD_PORTABLE78=1
-    set BUILD_PORTABLE259=1
+    set BUILD_PORTABLE=1
     set BUILD_VS=1
+    set BUILD_FSHARP_DATA_TYPEPROVIDERS=1
     set TEST_COMPILERUNIT=1
     set TEST_NET40_COREUNIT=1
-    set TEST_PORTABLE47_COREUNIT=1
-    set TEST_PORTABLE7_COREUNIT=1
-    set TEST_PORTABLE78_COREUNIT=1
-    set TEST_PORTABLE259_COREUNIT=1
+    set TEST_PORTABLE_COREUNIT=1
     set TEST_FSHARP_SUITE=1
     set TEST_FSHARPQA_SUITE=1
     set TEST_VS=0
@@ -128,33 +107,21 @@ if /i '%ARG%' == 'ci' (
 REM These divide 'ci' into three chunks which can be done in parallel
 
 if /i '%ARG%' == 'ci_part1' (
-    set BUILD_PORTABLE47=1
-    set BUILD_PORTABLE7=1
-    set BUILD_PORTABLE78=1
-    set BUILD_PORTABLE259=1
+    set BUILD_PORTABLE=1
     set BUILD_VS=1
+    set BUILD_FSHARP_DATA_TYPEPROVIDERS=1
     set TEST_COMPILERUNIT=1
     set TEST_NET40_COREUNIT=1
-    set TEST_PORTABLE47_COREUNIT=1
-    set TEST_PORTABLE7_COREUNIT=1
-    set TEST_PORTABLE78_COREUNIT=1
-    set TEST_PORTABLE259_COREUNIT=1
+    set TEST_PORTABLE_COREUNIT=1
     set TEST_VS=0
     set TEST_TAGS=
 )
 
 if /i '%ARG%' == 'ci_part2' (
-    set TEST_FSHARP_SUITE=1
-    set TEST_TAGS=
-)
-
-
-if /i '%ARG%' == 'ci_part3' (
-    set BUILD_PORTABLE47=1
-    set BUILD_PORTABLE7=1
-    set BUILD_PORTABLE78=1
-    set BUILD_PORTABLE259=1
+    set BUILD_PORTABLE=1
+    set BUILD_FSHARP_DATA_TYPEPROVIDERS=1
     set TEST_FSHARPQA_SUITE=1
+    set TEST_FSHARP_SUITE=1
     set TEST_TAGS=
 )
 
@@ -175,12 +142,8 @@ if /i '%ARG%' == 'debug' (
 )
 
 if /i '%ARG%' == 'build' (
-    set BUILD_PORTABLE47=1
-    set BUILD_PORTABLE7=1
-    set BUILD_PORTABLE78=1
-    set BUILD_PORTABLE259=1
+    set BUILD_PORTABLE=1
     set BUILD_VS=1
-
 )
 
 goto :EOF
@@ -192,17 +155,12 @@ REM after this point, ARG variable should not be used, use only BUILD_* or TEST_
 echo Build/Tests configuration:
 echo.
 echo BUILD_NET40=%BUILD_NET40%
-echo BUILD_PORTABLE47=%BUILD_PORTABLE47%
-echo BUILD_PORTABLE7=%BUILD_PORTABLE7%
-echo BUILD_PORTABLE78=%BUILD_PORTABLE78%
-echo BUILD_PORTABLE259=%BUILD_PORTABLE259%
+echo BUILD_PORTABLE=%BUILD_PORTABLE%
 echo BUILD_VS=%BUILD_VS%
+echo BUILD_FSHARP_DATA_TYPEPROVIDERS=%BUILD_FSHARP_DATA_TYPEPROVIDERS%
 echo.
 echo TEST_COMPILERUNIT=%TEST_COMPILERUNIT%
-echo TEST_PORTABLE47_COREUNIT=%TEST_PORTABLE47_COREUNIT%
-echo TEST_PORTABLE7_COREUNIT=%TEST_PORTABLE7_COREUNIT%
-echo TEST_PORTABLE78_COREUNIT=%TEST_PORTABLE78_COREUNIT%
-echo TEST_PORTABLE259_COREUNIT=%TEST_PORTABLE259_COREUNIT%
+echo TEST_PORTABLE_COREUNIT=%TEST_PORTABLE_COREUNIT%
 echo TEST_VS=%TEST_VS%
 echo TEST_FSHARP_SUITE=%TEST_FSHARP_SUITE%
 echo TEST_FSHARPQA_SUITE=%TEST_FSHARPQA_SUITE%
@@ -268,22 +226,21 @@ if '%BUILD_PROTO%' == '1' (
 %_msbuildexe% src/fsharp-compiler-build.proj /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: compiler build failed && goto :failure
 
-if '%BUILD_PORTABLE47%' == '1' (
+if '%BUILD_FSHARP_DATA_TYPEPROVIDERS%' == '1' (
+%_msbuildexe% src/fsharp-typeproviders-build.proj /p:Configuration=%BUILD_CONFIG%
+@if ERRORLEVEL 1 echo Error: type provider build failed && goto :failure
+)
+
+if '%BUILD_PORTABLE%' == '1' (
 %_msbuildexe% src/fsharp-library-build.proj /p:TargetFramework=portable47 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library portable47 build failed && goto :failure
-)
 
-if '%BUILD_PORTABLE7%' == '1' (
 %_msbuildexe% src/fsharp-library-build.proj /p:TargetFramework=portable7 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library portable7 build failed && goto :failure
-)
 
-if '%BUILD_PORTABLE78%' == '1' (
 %_msbuildexe% src/fsharp-library-build.proj /p:TargetFramework=portable78 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library portable78 build failed && goto :failure
-)
 
-if '%BUILD_PORTABLE259%' == '' (
 %_msbuildexe% src/fsharp-library-build.proj /p:TargetFramework=portable259 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library portable259 build failed && goto :failure
 )
@@ -297,22 +254,16 @@ if '%TEST_COREUNIT%' == '1' (
 @if ERRORLEVEL 1 echo Error: library unittests build failed && goto :failure
 )
 
-if '%TEST_PORTABLE47_COREUNIT%' == '1' (
+if '%TEST_PORTABLE_COREUNIT%' == '1' (
 %_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable47 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library unittests build failed portable47 && goto :failure
-)
 
-if '%TEST_PORTABLE7_COREUNIT%' == '1' (
 %_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable7 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library unittests build failed portable7 && goto :failure
-)
 
-if '%TEST_PORTABLE78_COREUNIT%' == '1' (
 %_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable78 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library unittests build failed portable78 && goto :failure
-)
 
-if '%TEST_PORTABLE259_COREUNIT%' == '1' (
 %_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable259 /p:Configuration=%BUILD_CONFIG%
 @if ERRORLEVEL 1 echo Error: library unittests build failed portable259 && goto :failure
 )
@@ -344,13 +295,13 @@ set FSHARP_TEST_SUITE_USE_NUNIT_RUNNER=true
 @if ERRORLEVEL 1 echo Error: fsharp cambridge tests for nunit failed && goto :failure
 
 call RunTests.cmd %BUILD_CONFIG_LOWER% fsharp %TEST_TAGS% 
-@if ERRORLEVEL 1 type testresults\fsharp_failures.log && echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% fsharp %CONF_CAMBRIDGE_SUITE%' failed && goto :failure
+@if ERRORLEVEL 1 type testresults\fsharp_failures.log && echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% fsharp %TEST_TAGS%' failed && goto :failure
 set FSHARP_TEST_SUITE_USE_NUNIT_RUNNER=
 )
 
 if '%TEST_FSHARPQA_SUITE%' == '1' (
 call RunTests.cmd %BUILD_CONFIG_LOWER% fsharpqa %TEST_TAGS% 
-@if ERRORLEVEL 1 type testresults\fsharpqa_failures.log && echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% fsharpqa %CONF_QA_SUITE%' failed && goto :failure
+@if ERRORLEVEL 1 type testresults\fsharpqa_failures.log && echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% fsharpqa %TEST_TAGS%' failed && goto :failure
 )
 
 if '%TEST_COMPILERUNIT%' == '1' (
@@ -363,27 +314,19 @@ call RunTests.cmd %BUILD_CONFIG_LOWER% coreunit %TEST_TAGS%
 @if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunit' failed && goto :failure
 )
 
-if '%TEST_PORTABLE47_COREUNIT%' == '1' (
+if '%TEST_PORTABLE_COREUNIT%' == '1' (
 call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable47 %TEST_TAGS% 
 @if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable47' failed && goto :failure
-)
 
-if '%TEST_PORTABLE7_COREUNIT%' == '1' (
 call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable7 %TEST_TAGS% 
 @if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable7' failed && goto :failure
-)
 
-if '%TEST_PORTABLE78_COREUNIT%' == '1' (
 call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable78 %TEST_TAGS% 
 @if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable78' failed && goto :failure
-)
 
-if '%TEST_PORTABLE259_COREUNIT%' == '1' (
 call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259 %TEST_TAGS% 
 @if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259' failed && goto :failure
-)
 
-if '%TEST_PORTABLE259_COREUNIT%' == '1' (
 call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259 %TEST_TAGS% 
 @if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259' failed && goto :failure
 )

src/fsharp/FSharp.Core.Unittests/FSharp.Core/Microsoft.FSharp.Collections/CollectionModulesConsistency.fs

@@ -1157,6 +1157,10 @@ let unfold<'a,'b when 'b : equality> f (start:'a) =
 [<Test>]
 let ``unfold is consistent`` () =
     Check.QuickThrowOnFailure unfold<int,int>
+
+[<Test; Category("Expensive")>]
+let ``unfold is consistent full`` () =
+    Check.QuickThrowOnFailure unfold<int,int>
     Check.QuickThrowOnFailure unfold<string,string>
     Check.QuickThrowOnFailure unfold<float,int>
     Check.QuickThrowOnFailure unfold<float,string>

src/update.fs

@@ -24,7 +24,7 @@ let private regQuery = WindowsPlatform.regQuery
 
 let private checkResult result =
     match result with
-    | CmdResult.ErrorLevel err -> let x = err, (sprintf "ERRORLEVEL %d" err) in Failure (RunError.ProcessExecError x)
+    | CmdResult.ErrorLevel (msg, err) -> Failure (RunError.ProcessExecError (msg, err, sprintf "ERRORLEVEL %d" err))
     | CmdResult.Success -> Success ()
 
 let updateCmd envVars args = processor {
@@ -91,12 +91,12 @@ let updateCmd envVars args = processor {
     // set NGEN64=%windir%\Microsoft.NET\Framework64\v4.0.30319\ngen.exe
     let NGEN64 = windir/"Microsoft.NET"/"Framework64"/"v4.0.30319"/"ngen.exe"
 
-    let checkResult = function CmdResult.ErrorLevel err -> Failure (sprintf "ERRORLEVEL %d" err) | CmdResult.Success -> Success ()
+    let checkResult = function CmdResult.ErrorLevel (msg, err) -> Failure (sprintf "%s. ERRORLEVEL %d" msg err) | CmdResult.Success -> Success ()
 
     let ngen32 = Commands.ngen exec NGEN32 >> checkResult
     let ngen64 = Commands.ngen exec NGEN64 >> checkResult
     let sn32 = exec SN32 >> checkResult
-    let sn64 = exec SN32 >> checkResult
+    let sn64 = exec SN64 >> checkResult
 
     // rem Disable strong-name validation for F# binaries built from open source that are signed with the microsoft key
     // %SN32% -Vr FSharp.Core,b03f5f7f11d50a3a
@@ -129,7 +129,8 @@ let updateCmd envVars args = processor {
             "VisualFSharp.Unittests";
             "VisualFSharp.Salsa" ]
         for a in all do
-            do! snExe (sprintf " -Vr %s,b03f5f7f11d50a3a" a) 
+            snExe (sprintf " -Vr %s,b03f5f7f11d50a3a" a)   |> ignore // ignore result - SN is not needed for tests to pass, and this fails without admin rights
+
         }
 
     do! strongName sn32
@@ -165,14 +166,15 @@ let updateCmd envVars args = processor {
         // "%NGEN32%" install "%BINDIR%\fsi.exe" /queue:1
         // "%NGEN32%" install "%BINDIR%\FSharp.Build.dll" /queue:1
         // "%NGEN32%" executeQueuedItems 1
-        do! ngen32 [binDir/"fsc.exe"; binDir/"fsi.exe"; binDir/"FSharp.Build.dll"]
+        ngen32 [binDir/"fsc.exe"; binDir/"fsi.exe"; binDir/"FSharp.Build.dll"] |> ignore // Ignore because may fail without admin rights
+
 
         // if /i "%PROCESSOR_ARCHITECTURE%"=="AMD64" (
         if processorArchitecture = AMD64 then
             // "%NGEN64%" install "%BINDIR%\fsiAnyCpu.exe" /queue:1
             // "%NGEN64%" install "%BINDIR%\FSharp.Build.dll" /queue:1
             // "%NGEN64%" executeQueuedItems 1
-            do! ngen64 [binDir/"fsiAnyCpu.exe"; binDir/"FSharp.Build.dll"]
+            ngen64 [binDir/"fsiAnyCpu.exe"; binDir/"FSharp.Build.dll"] |> ignore // Ignore because may fail without admin rights
         // )
     //:donengen
 

tests/RunTests.cmd

@@ -151,12 +151,12 @@ goto :EOF
 
 set FSHARP_TEST_SUITE_CONFIGURATION=%FLAVOR%
 
-set XMLFILE=FSharpNunit_Xml.xml
-set OUTPUTFILE=FSharpNunit_Output.log
-set ERRORFILE=FSharpNunit_Error.log
+set XMLFILE=%RESULTSDIR%\FSharpNunit_Xml.xml
+set OUTPUTFILE=%RESULTSDIR%\FSharpNunit_Output.log
+set ERRORFILE=%RESULTSDIR%\FSharpNunit_Error.log
 
-echo "%NUNIT3_CONSOLE%" --verbose "%FSCBINPATH%\..\..\net40\bin\FSharp.Tests.FSharp.dll" --framework:V4.0 %TTAGS_NUNIT_WHERE% --work:"%FSCBINPATH%"  --output:"%OUTPUTFILE%;format=nunit2" --err:"%ERRORFILE%" --result:"%XMLFILE%" 
-"%NUNIT3_CONSOLE%" --verbose "%FSCBINPATH%\..\..\net40\bin\FSharp.Tests.FSharp.dll" --framework:V4.0 %TTAGS_NUNIT_WHERE% --work:"%FSCBINPATH%"  --output:"%OUTPUTFILE%;format=nunit2" --err:"%ERRORFILE%" --result:"%XMLFILE%"
+echo "%NUNIT3_CONSOLE%" --verbose "%FSCBINPATH%\..\..\net40\bin\FSharp.Tests.FSharp.dll" --framework:V4.0 %TTAGS_NUNIT_WHERE% --work:"%FSCBINPATH%"  --output:"%OUTPUTFILE%;format=nunit2" --err:"%ERRORFILE%" --result:"%XMLFILE%;format=nunit2" 
+"%NUNIT3_CONSOLE%" --verbose "%FSCBINPATH%\..\..\net40\bin\FSharp.Tests.FSharp.dll" --framework:V4.0 %TTAGS_NUNIT_WHERE% --work:"%FSCBINPATH%"  --output:"%OUTPUTFILE%;format=nunit2" --err:"%ERRORFILE%" --result:"%XMLFILE%;format=nunit2"
 
 call :UPLOAD_XML "%XMLFILE%"
 goto :EOF

tests/config.fs

@@ -23,12 +23,6 @@ type private FSLibPaths =
       FSDATATPPATH : string
       FSCOREDLLVPREVPATH : string }
 
-
-let private checkResult result = 
-    match result with
-    | CmdResult.ErrorLevel err -> let x = err, (sprintf "ERRORLEVEL %d" err) in Failure (RunError.ProcessExecError x)
-    | CmdResult.Success -> Success ()
-
 // REM ===
 // REM === Find paths to shipped F# libraries referenced by clients
 // REM ===

tests/fsharp/Commands.fs

@@ -79,8 +79,9 @@ let ngen exec (ngenExe: FilePath) assemblies =
     List.concat [ queue; ["executeQueuedItems 1"] ]
     |> Seq.ofList
     |> Seq.map (fun args -> exec ngenExe args)
-    |> Seq.takeWhile (function ErrorLevel _ -> false | Ok -> true)
-    |> Seq.last
+    |> Seq.skipWhile (function ErrorLevel _ -> false | CmdResult.Success -> true)
+    |> Seq.tryHead
+    |> function None -> CmdResult.Success | Some res -> res
 
 let fsc exec (fscExe: FilePath) flags srcFiles =
     // "%FSC%" %fsc_flags% --define:COMPILING_WITH_EMPTY_SIGNATURE -o:tmptest2.exe tmptest2.mli tmptest2.ml