Skip to content

Commit 9c506c2

Browse files
michaelpjmichaelpj
committed
Use retry action, don't retry otherwise
This uses a action for retrying steps, which is a bit neater, and lets us more clearly specify what's going on, as well as controlling the number of retries independently. I think it's better to specifically use this on the test suites that we believe to be flaky, rather than adding a lot of noise by doing this on every test invocation.
1 parent 53f61b4 commit 9c506c2

File tree

1 file changed

+55
-32
lines changed

1 file changed

+55
-32
lines changed

‎.github/workflows/test.yml

+55-32
Original file line numberDiff line numberDiff line change
@@ -143,20 +143,35 @@ jobs:
143143
run: cabal test hls-graph --test-options="$TEST_OPTS"
144144

145145
- if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test
146+
uses: nick-fields/retry@v2
146147
name: Test ghcide
147-
# run the tests without parallelism to avoid running out of memory
148-
run: cabal test ghcide --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test ghcide --test-options="$TEST_OPTS"
148+
with:
149+
# This test is sometimes flaky
150+
max_attempts: 3
151+
# We don't really want to gate on a timeout
152+
timeout_minutes: 180
153+
shell: bash
154+
command: cabal test ghcide --test-options="$TEST_OPTS"
155+
new_command_on_retry: LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test ghcide --test-options="$TEST_OPTS"
149156

150157
- if: matrix.test
151158
name: Test hls-plugin-api
152-
run: cabal test hls-plugin-api --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-plugin-api --test-options="$TEST_OPTS"
159+
run: cabal test hls-plugin-api --test-options="$TEST_OPTS"
153160

154161
- if: matrix.test
162+
uses: nick-fields/retry@v2
155163
name: Test func-test suite
156164
env:
157165
HLS_TEST_EXE: hls
158166
HLS_WRAPPER_TEST_EXE: hls-wrapper
159-
run: cabal test func-test --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test func-test --test-options="$TEST_OPTS"
167+
with:
168+
# this test is sometimes flaky
169+
max_attempts: 3
170+
# We don't really want to gate on a timeout
171+
timeout_minutes: 180
172+
shell: bash
173+
command: cabal test func-test --test-options="$TEST_OPTS"
174+
new_command_on_retry: LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test func-test --test-options="$TEST_OPTS"
160175

161176
- if: matrix.test
162177
name: Test wrapper-test suite
@@ -167,116 +182,124 @@ jobs:
167182

168183
- if: matrix.test && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
169184
name: Test hls-brittany-plugin
170-
run: cabal test hls-brittany-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-brittany-plugin --test-options="$TEST_OPTS"
185+
run: cabal test hls-brittany-plugin --test-options="$TEST_OPTS"
171186

172187
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
188+
uses: nick-fields/retry@v2
173189
name: Test hls-refactor-plugin
174-
run: cabal test hls-refactor-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refactor-plugin --test-options="$TEST_OPTS"
190+
with:
191+
# This test is sometimes flaky
192+
max_attempts: 2
193+
# We don't really want to gate on a timeout
194+
timeout_minutes: 180
195+
shell: bash
196+
command: cabal test hls-refactor-plugin --test-options="$TEST_OPTS"
197+
new_command_on_retry: LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refactor-plugin --test-options="$TEST_OPTS"
175198

176199
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
177200
name: Test hls-floskell-plugin
178-
run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-floskell-plugin --test-options="$TEST_OPTS"
201+
run: cabal test hls-floskell-plugin --test-options="$TEST_OPTS"
179202

180203
- if: matrix.test
181204
name: Test hls-class-plugin
182-
run: cabal test hls-class-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-class-plugin --test-options="$TEST_OPTS"
205+
run: cabal test hls-class-plugin --test-options="$TEST_OPTS"
183206

184207
- if: matrix.test
185208
name: Test hls-pragmas-plugin
186-
run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-pragmas-plugin --test-options="$TEST_OPTS"
209+
run: cabal test hls-pragmas-plugin --test-options="$TEST_OPTS"
187210

188211
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
189212
name: Test hls-eval-plugin
190-
run: cabal test hls-eval-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-eval-plugin --test-options="$TEST_OPTS"
213+
run: cabal test hls-eval-plugin --test-options="$TEST_OPTS"
191214

192215
- if: matrix.test && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
193216
name: Test hls-haddock-comments-plugin
194-
run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS"
217+
run: cabal test hls-haddock-comments-plugin --test-options="$TEST_OPTS"
195218

196219
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
197220
name: Test hls-splice-plugin
198-
run: cabal test hls-splice-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-splice-plugin --test-options="$TEST_OPTS"
221+
run: cabal test hls-splice-plugin --test-options="$TEST_OPTS"
199222

200223
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
201224
name: Test hls-stylish-haskell-plugin
202-
run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS"
225+
run: cabal test hls-stylish-haskell-plugin --test-options="$TEST_OPTS"
203226

204227
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
205228
name: Test hls-ormolu-plugin
206-
run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-ormolu-plugin --test-options="$TEST_OPTS"
229+
run: cabal test hls-ormolu-plugin --test-options="$TEST_OPTS"
207230

208231
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
209232
name: Test hls-fourmolu-plugin
210-
run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS"
233+
run: cabal test hls-fourmolu-plugin --test-options="$TEST_OPTS"
211234

212235
- if: matrix.test && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
213236
name: Test hls-tactics-plugin test suite
214-
run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-tactics-plugin --test-options="$TEST_OPTS"
237+
run: cabal test hls-tactics-plugin --test-options="$TEST_OPTS"
215238

216239
- if: matrix.test
217240
name: Test hls-refine-imports-plugin test suite
218-
run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS"
241+
run: cabal test hls-refine-imports-plugin --test-options="$TEST_OPTS"
219242

220243
- if: matrix.test
221244
name: Test hls-explicit-imports-plugin test suite
222-
run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS"
245+
run: cabal test hls-explicit-imports-plugin --test-options="$TEST_OPTS"
223246

224247
- if: matrix.test
225248
name: Test hls-call-hierarchy-plugin test suite
226-
run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS"
249+
run: cabal test hls-call-hierarchy-plugin --test-options="$TEST_OPTS"
227250

228251
- if: matrix.test && matrix.os != 'windows-latest' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
229252
name: Test hls-rename-plugin test suite
230-
run: cabal test hls-rename-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-rename-plugin --test-options="$TEST_OPTS"
253+
run: cabal test hls-rename-plugin --test-options="$TEST_OPTS"
231254

232255
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
233256
name: Test hls-hlint-plugin test suite
234-
run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-hlint-plugin --test-options="$TEST_OPTS"
257+
run: cabal test hls-hlint-plugin --test-options="$TEST_OPTS"
235258

236259
- if: matrix.test && matrix.ghc != '9.0.1' && matrix.ghc != '9.0.2' && matrix.ghc != '9.2.4' && matrix.ghc != '9.2.5' && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
237260
name: Test hls-stan-plugin test suite
238-
run: cabal test hls-stan-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-stan-plugin --test-options="$TEST_OPTS"
261+
run: cabal test hls-stan-plugin --test-options="$TEST_OPTS"
239262

240263
- if: matrix.test
241264
name: Test hls-module-name-plugin test suite
242-
run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-module-name-plugin --test-options="$TEST_OPTS"
265+
run: cabal test hls-module-name-plugin --test-options="$TEST_OPTS"
243266

244267
- if: matrix.test
245268
name: Test hls-alternate-number-format-plugin test suite
246-
run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS"
269+
run: cabal test hls-alternate-number-format-plugin --test-options="$TEST_OPTS"
247270

248271
- if: matrix.test
249272
name: Test hls-qualify-imported-names-plugin test suite
250-
run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS"
273+
run: cabal test hls-qualify-imported-names-plugin --test-options="$TEST_OPTS"
251274

252275
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
253276
name: Test hls-code-range-plugin test suite
254-
run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-code-range-plugin --test-options="$TEST_OPTS"
277+
run: cabal test hls-code-range-plugin --test-options="$TEST_OPTS"
255278

256279
- if: matrix.test
257280
name: Test hls-change-type-signature test suite
258-
run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS"
281+
run: cabal test hls-change-type-signature-plugin --test-options="$TEST_OPTS"
259282

260283
- if: matrix.test && matrix.ghc != '9.4.2' && matrix.ghc != '9.4.3'
261284
name: Test hls-gadt-plugin test suit
262-
run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-gadt-plugin --test-options="$TEST_OPTS"
285+
run: cabal test hls-gadt-plugin --test-options="$TEST_OPTS"
263286

264287
- if: matrix.test
265288
name: Test hls-explicit-fixity-plugin test suite
266-
run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS"
289+
run: cabal test hls-explicit-fixity-plugin --test-options="$TEST_OPTS"
267290

268291
- if: matrix.test
269292
name: Test hls-explicit-record-fields-plugin test suite
270-
run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS"
293+
run: cabal test hls-explicit-record-fields-plugin --test-options="$TEST_OPTS"
271294

272295
## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions
273296
- if: matrix.test && matrix.ghc == '8.10.7'
274297
name: Test hls-cabal-fmt-plugin test suite
275-
run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS"
298+
run: cabal test hls-cabal-fmt-plugin --flag=isolateTests --test-options="$TEST_OPTS"
276299

277300
- if: matrix.test
278301
name: Test hls-cabal-plugin test suite
279-
run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS" || LSP_TEST_LOG_COLOR=0 LSP_TEST_LOG_MESSAGES=true LSP_TEST_LOG_STDERR=true cabal test hls-cabal-plugin --test-options="$TEST_OPTS"
302+
run: cabal test hls-cabal-plugin --test-options="$TEST_OPTS"
280303

281304
test_post_job:
282305
if: always()

0 commit comments

Comments
 (0)