Skip to content

Commit 5467df3

Browse files
dsymedsyme
committed
Merge pull request #937 from dsyme/fix-smoke
separate smoke and CI tests
2 parents 91c53e0 + c8a75ed commit 5467df3

18 files changed

Lines changed: 273 additions & 259 deletions

File tree

‎TESTGUIDE.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,8 @@ If all of these steps complete without issue, the test is considered to have pas
6666
These are all NUnit tests. You can execute these tests individually via the Visual Studio NUnit3 runner
6767
extension or the command line via `nunit3-console.exe`.
6868

69-
Note that for compatibility reasons, the IDE unit tests should be run in a 32-bit process, either `nunit3-console-x86.exe` or `nunit3-x86.exe`
70-
(todo: check if this comment still applies to NUnit3)
69+
Note that for compatibility reasons, the IDE unit tests should be run in a 32-bit process,
70+
using the '--x86' flag to `nunit3-console.exe`
7171

7272
### RunTests.cmd
7373

‎build.cmd‎

Lines changed: 117 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -13,35 +13,36 @@ echo Build and run a subset of test suites
1313
echo.
1414
echo Usage:
1515
echo.
16-
echo appveyor-build.cmd ^<all^|compiler^|pcls^|vs^|ci^|ci_part1^|ci_part2^|ci_part3^|build^|debug^>
16+
echo build.cmd ^<all^|build^|debug^|release^|compiler^|pcls^|vs^|ci^|ci_part1^|ci_part2^|ci_part3^>
1717
echo.
18-
echo No arguments default to 'ci' ( build all profiles, run all unit tests, cambridge Smoke, fsharpqa Smoke)
18+
echo No arguments default to 'build'
1919
echo.
2020
echo To specify multiple values, separate strings by comma
2121
echo.
2222
echo The example below run pcls, vs and qa:
2323
echo.
24-
echo appveyor-build.cmd pcls,vs,debug
24+
echo build.cmd pcls,vs,debug
2525
exit /b 1
2626

2727
:ARGUMENTS_OK
2828

29+
set BUILD_PROTO=0
2930
set BUILD_NET40=1
3031
set BUILD_PORTABLE47=0
3132
set BUILD_PORTABLE7=0
3233
set BUILD_PORTABLE78=0
3334
set BUILD_PORTABLE259=0
3435
set BUILD_VS=0
35-
set TEST_NET40=0
36-
set TEST_PORTABLE47=0
37-
set TEST_PORTABLE7=0
38-
set TEST_PORTABLE78=0
39-
set TEST_PORTABLE259=0
36+
set TEST_COMPILERUNIT=0
37+
set TEST_NET40_COREUNIT=0
38+
set TEST_PORTABLE47_COREUNIT=0
39+
set TEST_PORTABLE7_COREUNIT=0
40+
set TEST_PORTABLE78_COREUNIT=0
41+
set TEST_PORTABLE259_COREUNIT=0
4042
set TEST_VS=0
41-
set TEST_CAMBRIDGE_SUITE=0
42-
set CONF_CAMBRIDGE_SUITE=
43-
set TEST_QA_SUITE=0
44-
set CONF_QA_SUITE=
43+
set TEST_FSHARP_SUITE=0
44+
set TEST_TAGS=
45+
set TEST_FSHARPQA_SUITE=0
4546
set BUILD_CONFIG=Release
4647
set BUILD_CONFIG_LOWER=release
4748

@@ -61,26 +62,26 @@ goto :MAIN
6162
set ARG=%~1
6263

6364
if "%ARG%" == "1" if "%2" == "" (
64-
set ARG=ci
65+
set ARG=vuild
6566
)
6667

67-
if "%2" == "" if not "%ARG%" == "ci" goto :EOF
68+
if "%2" == "" if not "%ARG%" == "build" goto :EOF
6869

6970
echo Parse argument %ARG%
7071

7172
if /i '%ARG%' == 'compiler' (
72-
set TEST_NET40=1
73+
set TEST_COMPILERUNIT=1
7374
)
7475

7576
if /i '%ARG%' == 'pcls' (
7677
set BUILD_PORTABLE47=1
77-
set TEST_PORTABLE47=1
7878
set BUILD_PORTABLE7=1
79-
set TEST_PORTABLE7=1
8079
set BUILD_PORTABLE78=1
81-
set TEST_PORTABLE78=1
8280
set BUILD_PORTABLE259=1
83-
set TEST_PORTABLE259=1
81+
set TEST_PORTABLE47_COREUNIT=1
82+
set TEST_PORTABLE7_COREUNIT=1
83+
set TEST_PORTABLE78_COREUNIT=1
84+
set TEST_PORTABLE259_COREUNIT=1
8485
)
8586

8687
if /i '%ARG%' == 'vs' (
@@ -89,19 +90,20 @@ if /i '%ARG%' == 'vs' (
8990
)
9091

9192
if /i '%ARG%' == 'all' (
93+
set BUILD_PROTO=1
9294
set BUILD_PORTABLE47=1
9395
set BUILD_PORTABLE7=1
9496
set BUILD_PORTABLE78=1
9597
set BUILD_PORTABLE259=1
9698
set BUILD_VS=1
97-
set TEST_NET40=1
98-
set TEST_PORTABLE47=1
99-
set TEST_PORTABLE7=1
100-
set TEST_PORTABLE78=1
101-
set TEST_PORTABLE259=1
99+
set TEST_COMPILERUNIT=1
100+
set TEST_PORTABLE47_COREUNIT=1
101+
set TEST_PORTABLE7_COREUNIT=1
102+
set TEST_PORTABLE78_COREUNIT=1
103+
set TEST_PORTABLE259_COREUNIT=1
102104
set TEST_VS=1
103-
set TEST_CAMBRIDGE_SUITE=1
104-
set TEST_QA_SUITE=1
105+
set TEST_FSHARP_SUITE=1
106+
set TEST_FSHARPQA_SUITE=1
105107
)
106108

107109
REM Same as 'all' but smoke testing only
@@ -111,16 +113,17 @@ if /i '%ARG%' == 'ci' (
111113
set BUILD_PORTABLE78=1
112114
set BUILD_PORTABLE259=1
113115
set BUILD_VS=1
114-
set TEST_NET40=1
115-
set TEST_PORTABLE47=1
116-
set TEST_PORTABLE7=1
117-
set TEST_PORTABLE78=1
118-
set TEST_PORTABLE259=1
119-
set TEST_VS=1
120-
set TEST_CAMBRIDGE_SUITE=1
121-
set CONF_CAMBRIDGE_SUITE=Smoke
122-
set TEST_QA_SUITE=1
123-
set CONF_QA_SUITE=Smoke
116+
set TEST_COMPILERUNIT=1
117+
set TEST_NET40_COREUNIT=1
118+
set TEST_PORTABLE47_COREUNIT=1
119+
set TEST_PORTABLE7_COREUNIT=1
120+
set TEST_PORTABLE78_COREUNIT=1
121+
set TEST_PORTABLE259_COREUNIT=1
122+
set TEST_FSHARP_SUITE=1
123+
set TEST_FSHARPQA_SUITE=1
124+
set TEST_VS=0
125+
set TEST_TAGS=
126+
set TEST_TAGS2=CI
124127
)
125128

126129
REM These divide 'ci' into three chunks which can be done in parallel
@@ -131,30 +134,40 @@ if /i '%ARG%' == 'ci_part1' (
131134
set BUILD_PORTABLE78=1
132135
set BUILD_PORTABLE259=1
133136
set BUILD_VS=1
134-
set TEST_NET40=1
135-
set TEST_PORTABLE47=1
136-
set TEST_PORTABLE7=1
137-
set TEST_PORTABLE78=1
138-
set TEST_PORTABLE259=1
139-
set TEST_VS=1
137+
set TEST_COMPILERUNIT=1
138+
set TEST_NET40_COREUNIT=1
139+
set TEST_PORTABLE47_COREUNIT=1
140+
set TEST_PORTABLE7_COREUNIT=1
141+
set TEST_PORTABLE78_COREUNIT=1
142+
set TEST_PORTABLE259_COREUNIT=1
143+
set TEST_VS=0
144+
set TEST_TAGS=
145+
set TEST_TAGS2=CI
140146
)
141147

142148
if /i '%ARG%' == 'ci_part2' (
143-
set TEST_CAMBRIDGE_SUITE=1
144-
set CONF_CAMBRIDGE_SUITE=Smoke
149+
set TEST_FSHARP_SUITE=1
150+
set TEST_TAGS=
151+
set TEST_TAGS2=CI
145152
)
146153

147154

148155
if /i '%ARG%' == 'ci_part3' (
149-
set TEST_QA_SUITE=1
150-
set CONF_QA_SUITE=Smoke
156+
set TEST_FSHARPQA_SUITE=1
157+
set TEST_TAGS=
158+
set TEST_TAGS2=CI
151159
)
152160

153161
if /i '%ARG%' == 'smoke' (
154-
set TEST_CAMBRIDGE_SUITE=1
155-
set CONF_CAMBRIDGE_SUITE=Smoke
156-
set TEST_QA_SUITE=1
157-
set CONF_QA_SUITE=Smoke
162+
REM Smoke tests are a very small quick subset of tests
163+
164+
set TEST_COMPILERUNIT=0
165+
set TEST_NET40_COREUNIT=0
166+
set TEST_TAGS=Smoke
167+
168+
set TEST_FSHARP_SUITE=1
169+
set TEST_FSHARPQA_SUITE=0
170+
set TEST_TAGS2=Smoke
158171
)
159172

160173
if /i '%ARG%' == 'debug' (
@@ -163,14 +176,12 @@ if /i '%ARG%' == 'debug' (
163176
)
164177

165178
if /i '%ARG%' == 'build' (
166-
set TEST_NET40=0
167-
set TEST_PORTABLE47=0
168-
set TEST_PORTABLE7=0
169-
set TEST_PORTABLE78=0
170-
set TEST_PORTABLE259=0
171-
set TEST_VS=0
172-
set TEST_CAMBRIDGE_SUITE=0
173-
set TEST_QA_SUITE=0
179+
set BUILD_PORTABLE47=1
180+
set BUILD_PORTABLE7=1
181+
set BUILD_PORTABLE78=1
182+
set BUILD_PORTABLE259=1
183+
set BUILD_VS=1
184+
174185
)
175186

176187
goto :EOF
@@ -188,16 +199,15 @@ echo BUILD_PORTABLE78=%BUILD_PORTABLE78%
188199
echo BUILD_PORTABLE259=%BUILD_PORTABLE259%
189200
echo BUILD_VS=%BUILD_VS%
190201
echo.
191-
echo TEST_NET40=%TEST_NET40%
192-
echo TEST_PORTABLE47=%TEST_PORTABLE47%
193-
echo TEST_PORTABLE7=%TEST_PORTABLE7%
194-
echo TEST_PORTABLE78=%TEST_PORTABLE78%
195-
echo TEST_PORTABLE259=%TEST_PORTABLE259%
202+
echo TEST_COMPILERUNIT=%TEST_COMPILERUNIT%
203+
echo TEST_PORTABLE47_COREUNIT=%TEST_PORTABLE47_COREUNIT%
204+
echo TEST_PORTABLE7_COREUNIT=%TEST_PORTABLE7_COREUNIT%
205+
echo TEST_PORTABLE78_COREUNIT=%TEST_PORTABLE78_COREUNIT%
206+
echo TEST_PORTABLE259_COREUNIT=%TEST_PORTABLE259_COREUNIT%
196207
echo TEST_VS=%TEST_VS%
197-
echo TEST_CAMBRIDGE_SUITE=%TEST_CAMBRIDGE_SUITE%
198-
echo CONF_CAMBRIDGE_SUITE=%CONF_CAMBRIDGE_SUITE%
199-
echo TEST_QA_SUITE=%TEST_QA_SUITE%
200-
echo CONF_QA_SUITE=%CONF_QA_SUITE%
208+
echo TEST_FSHARP_SUITE=%TEST_FSHARP_SUITE%
209+
echo TEST_FSHARPQA_SUITE=%TEST_FSHARPQA_SUITE%
210+
echo TEST_TAGS=%TEST_TAGS%
201211
echo BUILD_CONFIG=%BUILD_CONFIG%
202212
echo BUILD_CONFIG_LOWER=%BUILD_CONFIG_LOWER%
203213
echo.
@@ -240,9 +250,15 @@ if not exist %_ngenexe% echo Error: Could not find ngen.exe. && goto :failure
240250
.\.nuget\NuGet.exe restore packages.config -PackagesDirectory packages -ConfigFile .nuget\nuget.config
241251
@if ERRORLEVEL 1 echo Error: Nuget restore failed && goto :failure
242252

243-
:: Build
253+
:: Build Proto
254+
if NOT EXIST Proto\net40\bin\fsc-proto.exe (
244255
%_msbuildexe% src\fsharp-proto-build.proj
245256
@if ERRORLEVEL 1 echo Error: compiler proto build failed && goto :failure
257+
)
258+
if '%BUILD_PROTO%' == '1' (
259+
%_msbuildexe% src\fsharp-proto-build.proj
260+
@if ERRORLEVEL 1 echo Error: compiler proto build failed && goto :failure
261+
)
246262

247263
%_ngenexe% install Proto\net40\bin\fsc-proto.exe
248264
@if ERRORLEVEL 1 echo Error: NGen of proto failed && goto :failure
@@ -273,30 +289,31 @@ if '%BUILD_PORTABLE259%' == '' (
273289
@if ERRORLEVEL 1 echo Error: library portable259 build failed && goto :failure
274290
)
275291

276-
if '%TEST_NET40%' == '1' (
292+
if '%TEST_COMPILERUNIT%' == '1' (
277293
%_msbuildexe% src/fsharp-compiler-unittests-build.proj /p:Configuration=%BUILD_CONFIG%
278294
@if ERRORLEVEL 1 echo Error: compiler unittests build failed && goto :failure
279-
295+
)
296+
if '%TEST_COREUNIT%' == '1' (
280297
%_msbuildexe% src/fsharp-library-unittests-build.proj /p:Configuration=%BUILD_CONFIG%
281298
@if ERRORLEVEL 1 echo Error: library unittests build failed && goto :failure
282299
)
283300

284-
if '%TEST_PORTABLE47%' == '1' (
301+
if '%TEST_PORTABLE47_COREUNIT%' == '1' (
285302
%_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable47 /p:Configuration=%BUILD_CONFIG%
286303
@if ERRORLEVEL 1 echo Error: library unittests build failed portable47 && goto :failure
287304
)
288305

289-
if '%TEST_PORTABLE7%' == '1' (
306+
if '%TEST_PORTABLE7_COREUNIT%' == '1' (
290307
%_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable7 /p:Configuration=%BUILD_CONFIG%
291308
@if ERRORLEVEL 1 echo Error: library unittests build failed portable7 && goto :failure
292309
)
293310

294-
if '%TEST_PORTABLE78%' == '1' (
311+
if '%TEST_PORTABLE78_COREUNIT%' == '1' (
295312
%_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable78 /p:Configuration=%BUILD_CONFIG%
296313
@if ERRORLEVEL 1 echo Error: library unittests build failed portable78 && goto :failure
297314
)
298315

299-
if '%TEST_PORTABLE259%' == '1' (
316+
if '%TEST_PORTABLE259_COREUNIT%' == '1' (
300317
%_msbuildexe% src/fsharp-library-unittests-build.proj /p:TargetFramework=portable259 /p:Configuration=%BUILD_CONFIG%
301318
@if ERRORLEVEL 1 echo Error: library unittests build failed portable259 && goto :failure
302319
)
@@ -306,11 +323,6 @@ if '%BUILD_VS%' == '1' (
306323
@if ERRORLEVEL 1 echo Error: VS integration build failed && goto :failure
307324
)
308325

309-
if '%TEST_VS%' == '1' (
310-
%_msbuildexe% vsintegration\fsharp-vsintegration-unittests-build.proj /p:Configuration=%BUILD_CONFIG%
311-
@if ERRORLEVEL 1 echo Error: VS integration unit tests build failed && goto :failure
312-
)
313-
314326
@echo on
315327
call src\update.cmd %BUILD_CONFIG_LOWER% -ngen
316328

@@ -326,51 +338,61 @@ call BuildTestTools.cmd %BUILD_CONFIG_LOWER%
326338
@if ERRORLEVEL 1 echo Error: 'BuildTestTools.cmd %BUILD_CONFIG_LOWER%' failed && goto :failure
327339

328340
@echo on
329-
if '%TEST_CAMBRIDGE_SUITE%' == '1' (
341+
if '%TEST_FSHARP_SUITE%' == '1' (
330342
set FSHARP_TEST_SUITE_USE_NUNIT_RUNNER=true
331343

332344
%_msbuildexe% fsharp\fsharp.tests.fsproj /p:Configuration=%BUILD_CONFIG%
333345
@if ERRORLEVEL 1 echo Error: fsharp cambridge tests for nunit failed && goto :failure
334346

335-
call RunTests.cmd %BUILD_CONFIG_LOWER% fsharp %CONF_CAMBRIDGE_SUITE%
347+
call RunTests.cmd %BUILD_CONFIG_LOWER% fsharp %TEST_TAGS2%
336348
@if ERRORLEVEL 1 type testresults\fsharp_failures.log && echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% fsharp %CONF_CAMBRIDGE_SUITE%' failed && goto :failure
337349
set FSHARP_TEST_SUITE_USE_NUNIT_RUNNER=
338350
)
339351

340-
if '%TEST_QA_SUITE%' == '1' (
341-
call RunTests.cmd %BUILD_CONFIG_LOWER% fsharpqa %CONF_QA_SUITE%
352+
if '%TEST_FSHARPQA_SUITE%' == '1' (
353+
call RunTests.cmd %BUILD_CONFIG_LOWER% fsharpqa %TEST_TAGS2%
342354
@if ERRORLEVEL 1 type testresults\fsharpqa_failures.log && echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% fsharpqa %CONF_QA_SUITE%' failed && goto :failure
343355
)
344356

345-
if '%TEST_NET40%' == '1' (
346-
call RunTests.cmd %BUILD_CONFIG_LOWER% compilerunit
357+
if '%TEST_COMPILERUNIT%' == '1' (
358+
call RunTests.cmd %BUILD_CONFIG_LOWER% compilerunit %TEST_TAGS%
347359
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% compilerunit' failed && goto :failure
360+
)
348361

349-
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunit
362+
if '%TEST_NET40_COREUNIT%' == '1' (
363+
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunit %TEST_TAGS%
350364
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunit' failed && goto :failure
351365
)
352366

353-
if '%TEST_PORTABLE47%' == '1' (
354-
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable47
367+
if '%TEST_PORTABLE47_COREUNIT%' == '1' (
368+
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable47 %TEST_TAGS%
355369
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable47' failed && goto :failure
356370
)
357371

358-
if '%TEST_PORTABLE7%' == '1' (
359-
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable7
372+
if '%TEST_PORTABLE7_COREUNIT%' == '1' (
373+
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable7 %TEST_TAGS%
360374
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable7' failed && goto :failure
361375
)
362376

363-
if '%TEST_PORTABLE78%' == '1' (
364-
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable78
377+
if '%TEST_PORTABLE78_COREUNIT%' == '1' (
378+
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable78 %TEST_TAGS%
365379
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable78' failed && goto :failure
366380
)
367381

368-
if '%TEST_PORTABLE259%' == '1' (
369-
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259
382+
if '%TEST_PORTABLE259_COREUNIT%' == '1' (
383+
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259 %TEST_TAGS%
384+
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259' failed && goto :failure
385+
)
386+
387+
if '%TEST_PORTABLE259_COREUNIT%' == '1' (
388+
call RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259 %TEST_TAGS%
370389
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259' failed && goto :failure
371390
)
372391

373-
rem tests for TEST_VS are not executed
392+
if '%TEST_VS%' == '1' (
393+
call RunTests.cmd %BUILD_CONFIG_LOWER% ideunit %TEST_TAGS%
394+
@if ERRORLEVEL 1 echo Error: 'RunTests.cmd %BUILD_CONFIG_LOWER% coreunitportable259' failed && goto :failure
395+
)
374396

375397
popd
376398

0 commit comments

Comments
 (0)