Skip to content

Commit bed3e5c

Browse files
ezyangezyang
committed
Greatly reduce the amount of build product we upload.
See haskell#4462 for the gory details. Main things about this commit: - New 'monolithic' flag on cabal-install, which combines all of the tests into a single binary. It's not very much code, and you don't pay for any of it on a release build. I quite like it. The one downside is that we can't also pull in Cabal test suites this way. - Env vars got moved into travis-common.sh - travis-script.sh now runs the cabal-tests tests, because we aren't sending enough build product over to do them on the second Travis run Signed-off-by: Edward Z. Yang <[email protected]>
1 parent db2d6e3 commit bed3e5c

13 files changed

+129
-71
lines changed

‎cabal-install/cabal-install.cabal

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,18 @@ flag lib
130130
Default: False
131131
manual: True
132132

133+
-- When we do CI, we build our binaries on one machine, and then
134+
-- ship them to another machine for testing. Because we use
135+
-- static linking (since it makes this sort of redeploy MUCH
136+
-- easier), if we build five executables, that means we
137+
-- need to ship ALL the Haskell libraries five times. That's
138+
-- a waste of space! A better strategy is to statically link
139+
-- everything into a single binary. That's what this flag does.
140+
flag monolithic
141+
description: Build cabal-install also with all of its test and support code. Used by our continuous integration.
142+
default: False
143+
manual: True
144+
133145
library
134146
ghc-options: -Wall -fwarn-tabs
135147
if impl(ghc >= 8.0)
@@ -442,6 +454,38 @@ executable cabal
442454
if flag(parsec)
443455
cpp-options: -DCABAL_PARSEC
444456

457+
if flag(monolithic)
458+
hs-source-dirs: tests
459+
other-modules:
460+
UnitTests
461+
MemoryUsageTests
462+
SolverQuickCheck
463+
IntegrationTests2
464+
cpp-options: -DMONOLITHIC
465+
build-depends:
466+
Cabal >= 2.1 && < 2.2,
467+
QuickCheck >= 2.8.2,
468+
array,
469+
async,
470+
bytestring,
471+
containers,
472+
deepseq,
473+
directory,
474+
edit-distance,
475+
filepath,
476+
mtl,
477+
network,
478+
network-uri,
479+
pretty-show,
480+
random,
481+
tagged,
482+
tar,
483+
tasty,
484+
tasty-hunit,
485+
tasty-quickcheck,
486+
time,
487+
zlib
488+
445489
if !(arch(arm) && impl(ghc < 7.6))
446490
ghc-options: -threaded
447491

@@ -459,7 +503,7 @@ Test-Suite unit-tests
459503
type: exitcode-stdio-1.0
460504
main-is: UnitTests.hs
461505
hs-source-dirs: tests
462-
ghc-options: -Wall -fwarn-tabs
506+
ghc-options: -Wall -fwarn-tabs -main-is UnitTests
463507
other-modules:
464508
UnitTests.Distribution.Client.ArbitraryInstances
465509
UnitTests.Distribution.Client.Targets
@@ -519,7 +563,7 @@ Test-Suite memory-usage-tests
519563
type: exitcode-stdio-1.0
520564
main-is: MemoryUsageTests.hs
521565
hs-source-dirs: tests
522-
ghc-options: -Wall -fwarn-tabs "-with-rtsopts=-M4M -K1K"
566+
ghc-options: -Wall -fwarn-tabs "-with-rtsopts=-M4M -K1K" -main-is MemoryUsageTests
523567
other-modules:
524568
UnitTests.Distribution.Solver.Modular.DSL
525569
UnitTests.Distribution.Solver.Modular.DSL.TestCaseUtils
@@ -549,7 +593,7 @@ Test-Suite solver-quickcheck
549593
type: exitcode-stdio-1.0
550594
main-is: SolverQuickCheck.hs
551595
hs-source-dirs: tests
552-
ghc-options: -Wall -fwarn-tabs
596+
ghc-options: -Wall -fwarn-tabs -main-is=SolverQuickCheck
553597
other-modules:
554598
UnitTests.Distribution.Solver.Modular.DSL
555599
UnitTests.Distribution.Solver.Modular.QuickCheck
@@ -579,7 +623,7 @@ test-suite integration-tests2
579623
type: exitcode-stdio-1.0
580624
main-is: IntegrationTests2.hs
581625
hs-source-dirs: tests
582-
ghc-options: -Wall -fwarn-tabs
626+
ghc-options: -Wall -fwarn-tabs -main-is IntegrationTests2
583627
other-modules:
584628
build-depends:
585629
base,

‎cabal-install/main/Main.hs

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,33 @@ import Data.Monoid (Any(..))
192192
import Control.Exception (SomeException(..), try)
193193
import Control.Monad (mapM_)
194194

195+
#ifdef MONOLITHIC
196+
import qualified UnitTests
197+
import qualified MemoryUsageTests
198+
import qualified SolverQuickCheck
199+
import qualified IntegrationTests2
200+
import qualified System.Environment as Monolithic
201+
#endif
202+
195203
-- | Entry point
196204
--
197205
main :: IO ()
206+
#ifdef MONOLITHIC
198207
main = do
208+
mb_exec <- Monolithic.lookupEnv "CABAL_INSTALL_MONOLITHIC_MODE"
209+
case mb_exec of
210+
Just "UnitTests" -> UnitTests.main
211+
Just "MemoryUsageTests" -> MemoryUsageTests.main
212+
Just "SolverQuickCheck" -> SolverQuickCheck.main
213+
Just "IntegrationTests2" -> IntegrationTests2.main
214+
Just s -> error $ "Unrecognized mode '" ++ show s ++ "' in CABAL_INSTALL_MONOLITHIC_MODE"
215+
Nothing -> main'
216+
#else
217+
main = main'
218+
#endif
219+
220+
main' :: IO ()
221+
main' = do
199222
-- Enable line buffering so that we can get fast feedback even when piped.
200223
-- This is especially important for CI and build systems.
201224
hSetBuffering stdout LineBuffering

‎cabal-install/tests/IntegrationTests2.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
-- For the handy instance IsString PackageIdentifier
66
{-# OPTIONS_GHC -fno-warn-orphans #-}
77

8-
module Main where
8+
module IntegrationTests2 where
99

1010
import Distribution.Client.DistDirLayout
1111
import Distribution.Client.ProjectConfig

‎cabal-install/tests/MemoryUsageTests.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
module Main where
1+
module MemoryUsageTests where
22

33
import Test.Tasty
44

‎cabal-install/tests/SolverQuickCheck.hs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
module Main where
1+
module SolverQuickCheck where
22

33
import Test.Tasty
44

‎cabal-install/tests/UnitTests.hs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
{-# LANGUAGE ScopedTypeVariables #-}
22

3-
module Main
4-
where
3+
module UnitTests where
54

65
import Test.Tasty
76

‎cabal.project

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
packages: Cabal/ cabal-testsuite/ cabal-install/
22
constraints: unix >= 2.7.1.0,
3-
cabal-install +lib
3+
cabal-install +lib +monolithic
44

55
-- Uncomment to allow picking up extra local unpacked deps:
66
--optional-packages: */

‎cabal.project.travis

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
-- Turn off parallelization to get good errors.
44
jobs: 1
55

6+
constraints: cabal-install +monolithic
7+
68
-- We vendor a copy of hackage-repo-tool so that we can
79
-- build it reliably. If we eventually get new-install
810
-- in the bootstrap, this can go away.

‎travis-common.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,14 @@ set -e
33
HACKAGE_REPO_TOOL_VERSION="0.1.1"
44
CABAL_VERSION="2.1.0.0"
55

6+
CABAL_STORE_DB="${HOME}/.cabal/store/ghc-${GHCVER}/package.db"
7+
CABAL_LOCAL_DB="${TRAVIS_BUILD_DIR}/dist-newstyle/packagedb/ghc-${GHCVER}"
8+
CABAL_BDIR="${TRAVIS_BUILD_DIR}/dist-newstyle/build/Cabal-${CABAL_VERSION}"
9+
CABAL_TESTSUITE_BDIR="${TRAVIS_BUILD_DIR}/dist-newstyle/build/cabal-testsuite-${CABAL_VERSION}"
10+
CABAL_INSTALL_BDIR="${TRAVIS_BUILD_DIR}/dist-newstyle/build/cabal-install-${CABAL_VERSION}"
11+
CABAL_INSTALL_SETUP="${CABAL_INSTALL_BDIR}/setup/setup"
12+
HACKAGE_REPO_TOOL_BDIR="${TRAVIS_BUILD_DIR}/dist-newstyle/build/hackage-repo-tool-${HACKAGE_REPO_TOOL_VERSION}"
13+
614
# ---------------------------------------------------------------------
715
# Timing / diagnostic output
816
# ---------------------------------------------------------------------

‎travis-script.sh

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,6 @@
1313

1414
. ./travis-common.sh
1515

16-
CABAL_STORE_DB="${HOME}/.cabal/store/ghc-${GHCVER}/package.db"
17-
CABAL_LOCAL_DB="${PWD}/dist-newstyle/packagedb/ghc-${GHCVER}"
18-
CABAL_BDIR="${PWD}/dist-newstyle/build/Cabal-${CABAL_VERSION}"
19-
CABAL_TESTSUITE_BDIR="${PWD}/dist-newstyle/build/cabal-testsuite-${CABAL_VERSION}"
20-
CABAL_INSTALL_BDIR="${PWD}/dist-newstyle/build/cabal-install-${CABAL_VERSION}"
21-
CABAL_INSTALL_SETUP="${CABAL_INSTALL_BDIR}/setup/setup"
22-
HACKAGE_REPO_TOOL_BDIR="${PWD}/dist-newstyle/build/hackage-repo-tool-${HACKAGE_REPO_TOOL_VERSION}"
2316
# --hide-successes uses terminal control characters which mess up
2417
# Travis's log viewer. So just print them all!
2518
TEST_OPTIONS=""
@@ -128,6 +121,15 @@ export CABAL_BUILDDIR="${CABAL_TESTSUITE_BDIR}"
128121
# both by Cabal and cabal-install
129122
timed cabal new-build $jobs cabal-testsuite:cabal-tests
130123

124+
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} -j3 $TEST_OPTIONS) || exit $?
125+
126+
# Redo the package tests with different versions of GHC
127+
if [ "x$TEST_OTHER_VERSIONS" = "xYES" ]; then
128+
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} $TEST_OPTIONS --with-ghc="/opt/ghc/7.0.4/bin/ghc")
129+
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} $TEST_OPTIONS --with-ghc="/opt/ghc/7.2.2/bin/ghc")
130+
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} $TEST_OPTIONS --with-ghc="/opt/ghc/head/bin/ghc")
131+
fi
132+
131133
unset CABAL_BUILDDIR
132134

133135
if [ "x$CABAL_LIB_ONLY" = "xYES" ]; then
@@ -147,18 +149,22 @@ if [ "x$DEBUG_EXPENSIVE_ASSERTIONS" = "xYES" ]; then
147149
CABAL_INSTALL_FLAGS=-fdebug-expensive-assertions
148150
fi
149151

150-
timed cabal new-build $jobs $CABAL_INSTALL_FLAGS \
151-
cabal-install:cabal \
152-
cabal-install:integration-tests2 \
153-
cabal-install:unit-tests \
154-
cabal-install:solver-quickcheck \
155-
cabal-install:memory-usage-tests
152+
# NB: For Travis, we do a *monolithic* build, which means all the
153+
# test suites are baked into the cabal binary
154+
timed cabal new-build $jobs $CABAL_INSTALL_FLAGS cabal-install:cabal
156155

157156
timed cabal new-build $jobs hackage-repo-tool
158157

159158
# Haddock
159+
# TODO: Figure out why this needs to be run before big tests
160160
(cd cabal-install && timed ${CABAL_INSTALL_SETUP} haddock --builddir=${CABAL_INSTALL_BDIR} ) || exit $?
161161

162+
# Tests need this
163+
timed ${CABAL_INSTALL_BDIR}/build/cabal/cabal update
164+
165+
# Big tests
166+
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} -j3 --skip-setup-tests --with-cabal ${CABAL_INSTALL_BDIR}/build/cabal/cabal --with-hackage-repo-tool ${HACKAGE_REPO_TOOL_BDIR}/build/hackage-repo-tool/hackage-repo-tool $TEST_OPTIONS) || exit $?
167+
162168
(cd cabal-install && timed cabal check) || exit $?
163169

164170
unset CABAL_BUILDDIR

‎travis/binaries/.travis.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ before_install:
1616
- export PATH=/opt/happy/1.19.5/bin:$PATH
1717
- export PATH=/opt/alex/3.1.7/bin:$PATH
1818
- ./travis-install.sh
19-
- mv .cabal $HOME
2019
script:
2120
- ./travis-test.sh
2221
notifications:

‎travis/binaries/travis-test.sh

Lines changed: 8 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,6 @@
22

33
. ./travis-common.sh
44

5-
CABAL_STORE_DB="${HOME}/.cabal/store/ghc-${GHCVER}/package.db"
6-
CABAL_LOCAL_DB="${PWD}/dist-newstyle/packagedb/ghc-${GHCVER}"
7-
CABAL_BDIR="${PWD}/dist-newstyle/build/Cabal-${CABAL_VERSION}"
8-
CABAL_TESTSUITE_BDIR="${PWD}/dist-newstyle/build/cabal-testsuite-${CABAL_VERSION}"
9-
CABAL_INSTALL_BDIR="${PWD}/dist-newstyle/build/cabal-install-${CABAL_VERSION}"
10-
CABAL_INSTALL_SETUP="${CABAL_INSTALL_BDIR}/setup/setup"
11-
HACKAGE_REPO_TOOL_BDIR="${PWD}/dist-newstyle/build/hackage-repo-tool-${HACKAGE_REPO_TOOL_VERSION}"
125
# --hide-successes uses terminal control characters which mess up
136
# Travis's log viewer. So just print them all!
147
TEST_OPTIONS=""
@@ -17,32 +10,17 @@ TEST_OPTIONS=""
1710
mkdir -p $(dirname $UPSTREAM_BUILD_DIR)
1811
ln -s $TRAVIS_BUILD_DIR $UPSTREAM_BUILD_DIR
1912

20-
# Touch package database cache files, so we don't complain they're
21-
# stale (Git doesn't preserve modification times, so we'll end
22-
# up with something wrong.)
23-
touch "$CABAL_STORE_DB/package.cache"
24-
touch "$CABAL_LOCAL_DB/package.cache"
25-
2613
# Run tests
27-
(timed ${CABAL_BDIR}/build/unit-tests/unit-tests $TEST_OPTIONS) || exit $?
14+
(timed Cabal/unit-tests $TEST_OPTIONS) || exit $?
2815

2916
if [ "x$PARSEC" = "xYES" ]; then
3017
# Parser unit tests
31-
(cd Cabal && timed ${CABAL_BDIR}/build/parser-tests/parser-tests $TEST_OPTIONS) || exit $?
18+
(cd Cabal && timed ./parser-tests $TEST_OPTIONS) || exit $?
3219

3320
# Test we can parse Hackage
34-
(cd Cabal && timed ${CABAL_BDIR}/build/parser-tests/parser-hackage-tests $TEST_OPTIONS) | tail || exit $?
21+
(cd Cabal && timed ./parser-hackage-tests $TEST_OPTIONS) | tail || exit $?
3522
fi
3623

37-
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} -j3 $TEST_OPTIONS) || exit $?
38-
39-
# Redo the package tests with different versions of GHC
40-
if [ "x$TEST_OTHER_VERSIONS" = "xYES" ]; then
41-
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} $TEST_OPTIONS --with-ghc="/opt/ghc/7.0.4/bin/ghc")
42-
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} $TEST_OPTIONS --with-ghc="/opt/ghc/7.2.2/bin/ghc")
43-
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} $TEST_OPTIONS --with-ghc="/opt/ghc/head/bin/ghc")
44-
fi
45-
4624
if [ "x$CABAL_LIB_ONLY" = "xYES" ]; then
4725
exit 0;
4826
fi
@@ -52,15 +30,12 @@ fi
5230
# ---------------------------------------------------------------------
5331

5432
# Update index
55-
(timed ${CABAL_INSTALL_BDIR}/build/cabal/cabal update) || exit $?
33+
(timed cabal-install/cabal update) || exit $?
5634

5735
# Run tests
58-
(timed ${CABAL_INSTALL_BDIR}/build/unit-tests/unit-tests $TEST_OPTIONS) || exit $?
59-
(cd cabal-install && timed ${CABAL_INSTALL_BDIR}/build/solver-quickcheck/solver-quickcheck $TEST_OPTIONS --quickcheck-tests=1000) || exit $?
60-
(timed ${CABAL_INSTALL_BDIR}/build/memory-usage-tests/memory-usage-tests $TEST_OPTIONS) || exit $?
36+
(timed env CABAL_INSTALL_MONOLITHIC_MODE=UnitTests cabal-install/cabal $TEST_OPTIONS) || exit $?
37+
(timed env CABAL_INSTALL_MONOLITHIC_MODE=MemoryUsageTests cabal-install/cabal $TEST_OPTIONS) || exit $?
6138

6239
# These need the cabal-install directory
63-
(cd cabal-install && timed ${CABAL_INSTALL_BDIR}/build/integration-tests2/integration-tests2 $TEST_OPTIONS) || exit $?
64-
65-
# Big tests
66-
(cd cabal-testsuite && timed ${CABAL_TESTSUITE_BDIR}/build/cabal-tests/cabal-tests --builddir=${CABAL_TESTSUITE_BDIR} -j3 --skip-setup-tests --with-cabal ${CABAL_INSTALL_BDIR}/build/cabal/cabal --with-hackage-repo-tool ${HACKAGE_REPO_TOOL_BDIR}/build/hackage-repo-tool/hackage-repo-tool $TEST_OPTIONS) || exit $?
40+
(cd cabal-install && timed env CABAL_INSTALL_MONOLITHIC_MODE=SolverQuickCheck ./cabal $TEST_OPTIONS --quickcheck-tests=1000) || exit $?
41+
(cd cabal-install && timed env CABAL_INSTALL_MONOLITHIC_MODE=IntegrationTests2 ./cabal $TEST_OPTIONS) || exit $?

‎travis/upload.sh

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@ COMMIT=${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT}
2222
# This is just to help you correlate the build to what it's for
2323
if [ "x$TRAVIS_PULL_REQUEST" != "xfalse" ]; then
2424
ORIGIN="${TRAVIS_REPO_SLUG}/pull/$TRAVIS_PULL_REQUEST"
25-
URL="https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST}"
25+
URL="pull/${TRAVIS_PULL_REQUEST}"
2626
else
2727
ORIGIN="${TRAVIS_REPO_SLUG}/${TRAVIS_BRANCH}"
28-
URL="https://github.com/${TRAVIS_REPO_SLUG}/commits/${TRAVIS_BRANCH}"
28+
URL="commits/${TRAVIS_BRANCH}"
2929
fi
3030

3131
# Git will complain if these fields don't work when committing,
@@ -59,21 +59,23 @@ if [ "x$GHCVER" = "x7.8.4" ] && [ "x$TRAVIS_OS_NAME" = "xosx" ]; then
5959
echo "osx_image: xcode6.4" >> .travis.yml
6060
fi
6161

62+
# Make directory layout
63+
mkdir Cabal
64+
mkdir cabal-install
65+
cp -R $TRAVIS_BUILD_DIR/Cabal/tests Cabal
66+
cp -R $TRAVIS_BUILD_DIR/cabal-install/tests cabal-install
6267
# Install all of the necessary files for testing
6368
cp $TRAVIS_BUILD_DIR/travis-install.sh .
6469
cp $TRAVIS_BUILD_DIR/travis-common.sh .
65-
cp -R $HOME/.cabal .
66-
# Index files are too big for Git
67-
rm -fv .cabal/packages/hackage.haskell.org/00-index*
68-
rm -fv .cabal/packages/hackage.haskell.org/01-index*
69-
rm -fv .cabal/packages/hackage.haskell.org/*.json
70-
cp -R $TRAVIS_BUILD_DIR/dist-newstyle .
71-
# Test files for test suites that rely on them
72-
cp -R $TRAVIS_BUILD_DIR/cabal-testsuite .
73-
mkdir Cabal
74-
cp -R $TRAVIS_BUILD_DIR/Cabal/tests Cabal
75-
mkdir cabal-install
76-
cp -R $TRAVIS_BUILD_DIR/cabal-install/tests cabal-install
70+
# The binaries to test (statically linked, of course!)
71+
cp ${CABAL_BDIR}/build/unit-tests/unit-tests Cabal
72+
if [ "x$PARSEC" = "xYES" ]; then
73+
cp ${CABAL_BDIR}/build/parser-tests/parser-tests Cabal
74+
cp ${CABAL_BDIR}/build/parser-hackage-tests/parser-hackage-tests Cabal
75+
fi
76+
if [ "x$CABAL_LIB_ONLY" != "xYES" ]; then
77+
cp ${CABAL_INSTALL_BDIR}/build/cabal/cabal cabal-install
78+
fi
7779

7880
# Add, commit, push
7981
git add .

0 commit comments

Comments
 (0)