test: run in parallel across 4 workers, 30s -> 13s

running test suite in parallel before we get too far

we should also monitor pytest-dev/pytest-xdist#229
because right now one cannot run tests in random order as well as
run in parallel

until then, I've added another makefile target that can run the tests
in parallel, and have left that running in CI.

Author: redshiftzero

Makefile

@@ -31,7 +31,14 @@ clean:  ## Clean the workspace of generated resources
 TESTS ?= tests
 TESTOPTS ?= -v
 .PHONY: test
-test: ## Run the application tests
+test: ## Run the application tests in parallel (for rapid development)
+	@TEST_CMD="python -m pytest -v -n 4 --cov-config .coveragerc --cov-report html --cov-report term-missing --cov=securedrop_client --cov-fail-under 100 $(TESTOPTS) $(TESTS)" ; \
+		if command -v xvfb-run > /dev/null; then \
+		xvfb-run $$TEST_CMD ; else \
+		$$TEST_CMD ; fi
+
+.PHONY: test-random
+test-random: ## Run the application tests in random order
 	@TEST_CMD="python -m pytest -v --random-order-bucket=global --cov-config .coveragerc --cov-report html --cov-report term-missing --cov=securedrop_client --cov-fail-under 100 $(TESTOPTS) $(TESTS)" ; \
 		if command -v xvfb-run > /dev/null; then \
 		xvfb-run $$TEST_CMD ; else \
@@ -60,7 +67,7 @@ bandit: ## Run bandit with medium level excluding test-related folders
 	bandit -ll --recursive . --exclude ./tests,./.venv
 
 .PHONY: check
-check: clean bandit lint mypy test ## Run the full CI test suite
+check: clean bandit lint mypy test-random ## Run the full CI test suite
 
 .PHONY: update-pip-requirements
 update-pip-requirements: ## Updates all Python requirements files via pip-compile.

README.md

@@ -113,6 +113,8 @@ To run everything, run:
 make check
 ```
 
+To individually run the unit tests, run `make test` to run the suite in parallel (fast), or run `make test-random` to run the tests in random order (slower, but this is what `make check` runs and what runs in CI).
+
 ## Environments
 
 The quickest way to get started with running the client is to use the [developer environment](#developer-environment) that [runs against a test server running in a local docker container](#running-against-a-test-server). This differs from a staging or production environment where the client receives and sends requests over Tor. Things are a lot snappier in the developer environment and can sometimes lead to a much different user experience, which is why it is important to do end-to-end testing in Qubes using the [staging environment](#staging-environment), especially if you are modifying code paths involving how we handle server requests and responses.

dev-requirements.in

@@ -9,14 +9,15 @@ more-itertools==4.3.0
 mypy==0.701
 mypy-extensions==0.4.1
 pip-tools==3.1.0
-pluggy==0.8.0
+pluggy==0.13.0
 py==1.7.0
 pycodestyle==2.4.0
 pyflakes==2.0.0
 PyQt5==5.10.1
-pytest==3.9.3
-pytest-cov==2.6.0
+pytest==5.2.1
+pytest-cov==2.8.1
 pytest-mock==1.10.0
 pytest-random-order==1.0.0
+pytest-xdist==1.30.0
 sip==4.19.8
 typed-ast==1.3.4

dev-requirements.txt

@@ -2,10 +2,14 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes --output-file=dev-requirements.txt dev-requirements.in requirements.in
+#    pip-compile --generate-hashes --output-file dev-requirements.txt requirements.in dev-requirements.in
 #
 alembic==1.0.2 \
     --hash=sha256:04bcb970ca8659c3607ddd8ffd86cc9d6a99661c9bc590955e8813c66bfa582b
+apipkg==1.5 \
+    --hash=sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6 \
+    --hash=sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c \
+    # via execnet
 arrow==0.12.1 \
     --hash=sha256:a558d3b7b6ce7ffc74206a86c147052de23d3d4ef0e17c210dd478c53575c4cd
 atomicwrites==1.2.1 \
@@ -57,12 +61,20 @@ coverage==4.5.1 \
     --hash=sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6 \
     --hash=sha256:f05a636b4564104120111800021a92e43397bc12a5c72fed7036be8556e0029e \
     --hash=sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80
+execnet==1.7.1 \
+    --hash=sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50 \
+    --hash=sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547 \
+    # via pytest-xdist
 flake8==3.6.0 \
     --hash=sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670 \
     --hash=sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2
 idna==2.7 \
     --hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
     --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16
+importlib-metadata==0.23 \
+    --hash=sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26 \
+    --hash=sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af \
+    # via pluggy, pytest
 mako==1.0.7 \
     --hash=sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae
 markupsafe==1.0 \
@@ -89,15 +101,19 @@ mypy==0.701 \
     --hash=sha256:bbf643528e2a55df2c1587008d6e3bda5c0445f1240dfa85129af22ae16d7a9a \
     --hash=sha256:c46ab3438bd21511db0f2c612d89d8344154c0c9494afc7fbc932de514cf8d15 \
     --hash=sha256:f7a83d6bd805855ef83ec605eb01ab4fa42bcef254b13631e451cbb44914a9b0
+packaging==19.2 \
+    --hash=sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47 \
+    --hash=sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108 \
+    # via pytest
 pathlib2==2.3.2 \
     --hash=sha256:8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83 \
     --hash=sha256:d1aa2a11ba7b8f7b21ab852b1fb5afb277e1bb99d5dfc663380b5015c0d80c5a
 pip-tools==3.1.0 \
     --hash=sha256:31b43e5f8d605fc84f7506199025460abcb98a29d12cc99db268f73e39cf55e5 \
     --hash=sha256:b1ceca03b4a48346b2f6870565abb09d8d257d5b1524b4c6b222185bf26c3870
-pluggy==0.8.0 \
-    --hash=sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095 \
-    --hash=sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f
+pluggy==0.13.0 \
+    --hash=sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6 \
+    --hash=sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34
 py==1.7.0 \
     --hash=sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694 \
     --hash=sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6
@@ -107,23 +123,33 @@ pycodestyle==2.4.0 \
 pyflakes==2.0.0 \
     --hash=sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49 \
     --hash=sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae
+pyparsing==2.4.2 \
+    --hash=sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80 \
+    --hash=sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4 \
+    # via packaging
 pyqt5==5.10.1 \
     --hash=sha256:1e652910bd1ffd23a3a48c510ecad23a57a853ed26b782cd54b16658e6f271ac \
     --hash=sha256:4db7113f464c733a99fcb66c4c093a47cf7204ad3f8b3bda502efcc0839ac14b \
     --hash=sha256:9c17ab3974c1fc7bbb04cc1c9dae780522c0ebc158613f3025fccae82227b5f7 \
     --hash=sha256:f6035baa009acf45e5f460cf88f73580ad5dc0e72330029acd99e477f20a5d61
-pytest-cov==2.6.0 \
-    --hash=sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7 \
-    --hash=sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762
+pytest-cov==2.8.1 \
+    --hash=sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b \
+    --hash=sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626
+pytest-forked==1.1.1 \
+    --hash=sha256:e2d46f319c8063a3a0536b18f9cdea6eea3bc9fe2cb16c94e1d6fad3abc37300 \
+    # via pytest-xdist
 pytest-mock==1.10.0 \
     --hash=sha256:53801e621223d34724926a5c98bd90e8e417ce35264365d39d6c896388dcc928 \
     --hash=sha256:d89a8209d722b8307b5e351496830d5cc5e192336003a485443ae9adeb7dd4c0
 pytest-random-order==1.0.0 \
     --hash=sha256:23881501819fdd03e7b6885b0aae83d212f5689851ded7b5ec2bdb00ad1c6c7f \
     --hash=sha256:7844b4c213bd18a3102dcc735010a0b44746801b97222d70bb4ea7d4ad535a43
-pytest==3.9.3 \
-    --hash=sha256:a9e5e8d7ab9d5b0747f37740276eb362e6a76275d76cebbb52c6049d93b475db \
-    --hash=sha256:bf47e8ed20d03764f963f0070ff1c8fda6e2671fc5dd562a4d3b7148ad60f5ca
+pytest-xdist==1.30.0 \
+    --hash=sha256:5d1b1d4461518a6023d56dab62fb63670d6f7537f23e2708459a557329accf48 \
+    --hash=sha256:a8569b027db70112b290911ce2ed732121876632fb3f40b1d39cd2f72f58b147
+pytest==5.2.1 \
+    --hash=sha256:7e4800063ccfc306a53c461442526c5571e1462f61583506ce97e4da6a1d88c8 \
+    --hash=sha256:ca563435f4941d0cb34767301c27bc65c510cb82e90b9ecf9cb52dc2c63caaa0
 python-dateutil==2.7.5 \
     --hash=sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93 \
     --hash=sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02
@@ -132,8 +158,8 @@ python-editor==1.0.3 \
 requests==2.20.0 \
     --hash=sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c \
     --hash=sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279
-securedrop-sdk==0.0.10 \
-    --hash=sha256:07b9f9d91b26e2396ff5eb63b5e990ac969de57bbfb7c075d99ac5e269198cb0
+securedrop-sdk==0.0.11 \
+    --hash=sha256:89a454a45eac15db702e06d24f5e4be64b8d033a5546aff056de762ecfc5f72a
 sip==4.19.8 \
     --hash=sha256:09f9a4e6c28afd0bafedb26ffba43375b97fe7207bd1a0d3513f79b7d168b331 \
     --hash=sha256:105edaaa1c8aa486662226360bd3999b4b89dd56de3e314d82b83ed0587d8783 \
@@ -176,7 +202,11 @@ typed-ast==1.3.4 \
 urllib3==1.24.3 \
     --hash=sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4 \
     --hash=sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb
-
-# WARNING: The following packages were not pinned, but pip requires them to be
-# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
-# setuptools==41.0.1        # via flake8, pytest
+wcwidth==0.1.7 \
+    --hash=sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e \
+    --hash=sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c \
+    # via pytest
+zipp==0.6.0 \
+    --hash=sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e \
+    --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335 \
+    # via importlib-metadata