IntersectMBO · iohk-bors · Nov 8, 2022 · Oct 13, 2022
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -88,9 +88,6 @@ jobs:
         cabal --version
         echo "cabal-store=$(dirname $(cabal --help | tail -1 | tr -d ' '))\\store" >> $GITHUB_OUTPUT
 
-    - name: Update Hackage index
-      run: cabal update
-
     - name: Set cache version
       run: echo "CACHE_VERSION=20220919" >> $GITHUB_ENV
 
@@ -134,6 +131,9 @@ jobs:
       run: |
         cp .github/workflows/cabal.project.local.Windows cabal.project.local
 
+    - name: Update Hackage and CHaP
+      run: cabal update
+
     - name: Record dependencies
       id: record-deps
       run: |

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -38,3 +38,64 @@ out=$(pwd) genericBuild
 ```
 
 Build logs are written to stdout and artifacts written to the current directory.
+
+## Releasing packages to CHaP
+
+When new versions of the packages are released, they should be included in [CHaP](https://github.com/input-output-hk/cardano-haskell-packages).
+See the CHaP README for [instructions](https://github.com/input-output-hk/cardano-haskell-packages#-from-github).
+
+## Updating dependencies
+
+Our Haskell packages come from two package repositories:
+- Hackage
+- [CHaP](https://github.com/input-output-hk/cardano-haskell-packages) (which is essentially another Hackage)
+
+The "index state" of each repository is pinned to a particular time in
+`cabal.project`.  This tells Cabal to treat the repository as if it was
+the specified time, ensuring reproducibility.  If you want to use a package
+version from repository X which was added after the pinned index state
+time, you need to bump the index state for X.  This is not a big deal,
+since all it does is change what packages `cabal` considers to be available
+when doing solving, but it will change what package versions cabal picks
+for the plan, and so will likely result in significant recompilation, and
+potentially some breakage.  That typically just means that we need to fix
+the breakage (increasing the lower-bound on the problematic package if fix 
+is not backward compatible), or delay that work and instead decrease the 
+upper-bound on the problematic package for now.
+
+Note that `cabal`'s own persistent state includes which index states it is 
+aware of, so when you bump the pinned index state you may need to
+call `cabal update` in order for `cabal` to be happy.
+
+The Nix code which builds our packages also needs some information relating 
+to the index-state. This information needs to be new enough to include 
+the index-state specified in `cabal.project`. The information is represented 
+by inputs managed by `niv`:
+You can update these by running:
+- `niv update hackage.nix` for Hackage
+- `niv update CHaP` for CHaP
+
+If you fail to do this you may get an error like this from Nix:
+```
+error: Unknown index-state 2021-08-08T00:00:00Z, the latest index-state I know about is 2021-08-06T00:00:00Z. You may need to update to a newer hackage.nix.
+```
+
+### Use of `source-repository-package`s
+
+We *can* use Cabal's `source-repository-package` mechanism to pull in
+un-released package versions. This can be useful when debugging/developing
+across different repositories. However, we should not release our packages
+to CHaP while we depend on a `source-repository-package` since downstream
+consumers would not be able to build such package.
+
+If we are stuck in a situation where we need a long-running fork of a
+package, we should release it to CHaP instead (see the
+https://github.com/input-output-hk/cardano-haskell-packages[CHaP README]
+for more).
+
+If you do add a temporary `source-repository-package` stanza, you need to
+provide a `--sha256` comment in `cabal.project` so that Nix knows the hash
+of the content. There are two relatively straightforward ways to do this:
+
+1. The TOFU approach: put in the wrong hash and then Nix will tell you the correct one, which you can copy in.
+2. Calculate the hash with `nix-shell -p nix-prefetch-git --run 'nix-prefetch-git <URL> <COMMIT_HASH>'`
diff --git a/cabal.project b/cabal.project
@@ -1,4 +1,21 @@
+-- Custom repository for cardano haskell packages, see CONTRIBUTING for more
+repository cardano-haskell-packages
+  url: https://input-output-hk.github.io/cardano-haskell-packages
+  secure: True
+  root-keys:
+    3e0cce471cf09815f930210f7827266fd09045445d65923e6d0238a6cd15126f
+    443abb7fb497a134c343faf52f0b659bd7999bc06b7f63fa76dc99d631f9bea1
+    a86a1f6ce86c449c46666bda44268677abf29b5b2d2eb5ec7af903ec2f117a82
+    bcec67e8e99cabfa7764d75ad9b158d72bfacf70ca1d0ec8bc6b4406d1bf8413
+    c00aae8461a256275598500ea0e187588c35a5d5d7454fb57eac18d9edb86a56
+    d4a35cd3121aa00d18544bb0ac01c3e1691d618f462c46129271bccf39f7e8ee
+
+-- See CONTRIBUTING for some Nix commands you will need to run if you
+-- update either of these.
+-- Bump this if you need newer packages from Hackage
 index-state: 2022-07-01T00:00:00Z
+-- Bump this if you need newer packages from CHaP
+index-state: cardano-haskell-packages 2022-11-07T00:00:00Z
 
 packages: ./ouroboros-network-testing
           ./monoidal-synchronisation
@@ -22,18 +39,7 @@ packages: ./ouroboros-network-testing
           ./ntp-client
           ./cardano-client
 
-constraints:
-    ip < 1.5
-  , hedgehog >= 1.0
-  , bimap >= 0.4.0
-  , ListLike >= 4.7.3
-  , witherable >= 0.4
-
-  -- Plutus dependency
-  , algebraic-graphs < 0.7
-
-package Win32-network
-  tests: True
+tests: True
 
 package io-classes
   flags: +asserts
@@ -44,184 +50,38 @@ package strict-stm
 package network-mux
   flags: +asserts
 
-package ouroboros-network-testing
-  tests: True
-
-package mux
-  tests: True
-
 package ouroboros-network
-  tests: True
   flags: +asserts +cddl
 
-
 package ouroboros-consensus
   flags: +asserts
 
-package ouroboros-consensus-test
-  tests: True
-
 package ouroboros-consensus-mock
   flags: +asserts
 
-package ouroboros-consensus-mock-test
-  tests: True
-
 package ouroboros-consensus-byron
   flags: +asserts
 
-package ouroboros-consensus-byron-test
-  tests: True
-
 package ouroboros-consensus-shelley
   flags: +asserts
 
-package ouroboros-consensus-shelley-test
-  tests: True
-
 package ouroboros-consensus-cardano
   flags: +asserts
 
-package ouroboros-consensus-cardano-test
-  tests: True
-
-package ouroboros-consensus-cardano-tools
-  tests: True
-
-
-package cardano-binary
-  tests: False
-
-package cardano-crypto
-  tests: False
-
-package cardano-crypto-class
-  tests: False
-
-package cardano-crypto-wrapper
-  tests: False
-
-package cardano-prelude-test
-  tests: False
-
-package contra-tracer
-  tests: False
-
-package byron-spec-chain
-  tests: False
-
-package byron-spec-ledger
-  tests: False
-
-package cardano-ledger-byron
-  tests: False
-
-package cardano-ledger-shelley-ma-test
-  tests: False
-
-package cardano-ledger-byron-test
-  tests: False
-
-package goblins
-  tests: False
-
-package shelley-spec-ledger-test
-  tests: False
-
-package cardano-ledger-alonzo-test
-  tests: False
-
-package shelley-spec-non-integral
-  tests: False
-
-package small-steps
-  tests: False
-
-package small-steps-test
-  tests: False
-
-package plutus-tx
-  tests: False
-
-package plutus-core
-  tests: False
-
-package prettyprinter-configurable
-  tests: False
-
-allow-newer:
-  size-based:template-haskell
-
 constraints:
+  -- CONSTRAINTS FOR UPSTREAM CARDANO PACKAGES
+  -- These should all be constraints in upstream packages, but are currently missing.
+  -- We should a) fix these upstream and b) revise the constraints on the released
+  -- version in CHaP.
+  -- TODO Unfortunately revisions in CHaP don't work yet, revisit this when we fix it.
+  -- See https://github.com/input-output-hk/haskell.nix/issues/1675
+
   -- bizarre issue: in earlier versions they define their own 'GEq', in newer
   -- ones they reuse the one from 'some', but there isn't e.g. a proper version
   -- constraint from dependent-sum-template (which is the library we actually use).
   , dependent-sum > 0.6.2.0
-
--- ---------------------------------------------------------
--- The "cabal" wrapper script provided by nix-shell will cut off / restore the remainder of this file
--- in order to force usage of nix provided dependencies for `source-repository-package`s.
--- --------------------------- 8< --------------------------
--- Please do not put any `source-repository-package` clause above this line.
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/io-sim
-  tag: 0df93fe773e9b66ce35f3637128533e5b0029558
-  --sha256: 0vcafginvx4vcd78brihan8b8s436qihrl6h3swsgh86vbfp2r27
-  subdir:
-    io-classes
-    io-sim
-    strict-stm
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/typed-protocols
-  tag: c5e87c9ba2854966794f04c5c2887faa09bf2214
-  --sha256: 06iqdgl97rms9mk56kn4g445ppi47jsshfn5gl1f9cr1zvlqhcrv
-  subdir:
-    typed-protocols
-    typed-protocols-cborg
-    typed-protocols-examples
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/Win32-network
-  tag: 3825d3abf75f83f406c1f7161883c438dac7277d
-  --sha256: 19wahfv726fa3mqajpqdqhnl9ica3xmf68i254q45iyjcpj1psqx
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/iohk-monitoring-framework
-  tag: 066f7002aac5a0efc20e49643fea45454f226caa
-  --sha256: 0s6x4in11k5ba7nl7la896g28sznf9185xlqg9c604jqz58vj9nj
-  subdir: contra-tracer
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/cardano-prelude
-  tag: 533aec85c1ca05c7d171da44b89341fb736ecfe5
-  --sha256: 0z2y3wzppc12bpn9bl48776ms3nszw8j58xfsdxf97nzjgrmd62g
-  subdir:
-    cardano-prelude
-    cardano-prelude-test
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/cardano-base
-  tag: 631cb6cf1fa01ab346233b610a38b3b4cba6e6ab
-  --sha256: 0944wg2nqazmhlmsynwgdwxxj6ay0hb9qig9l128isb2cjia0hlp
-  subdir:
-    base-deriving-via
-    binary
-    binary/test
-    cardano-crypto-class
-    cardano-crypto-praos
-    cardano-crypto-tests
-    measures
-    orphans-deriving-via
-    slotting
-    strict-containers
+  -- plutus-core: needs a constraint here, fixed on plutus master but not in the released version
+  , algebraic-graphs < 0.7
 
 source-repository-package
   type: git
@@ -253,6 +113,32 @@ source-repository-package
     libs/small-steps-test
     libs/vector-map
 
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-prelude
+  tag: 533aec85c1ca05c7d171da44b89341fb736ecfe5
+  --sha256: 0z2y3wzppc12bpn9bl48776ms3nszw8j58xfsdxf97nzjgrmd62g
+  subdir:
+    cardano-prelude
+    cardano-prelude-test
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/cardano-base
+  tag: 631cb6cf1fa01ab346233b610a38b3b4cba6e6ab
+  --sha256: 0944wg2nqazmhlmsynwgdwxxj6ay0hb9qig9l128isb2cjia0hlp
+  subdir:
+    base-deriving-via
+    binary
+    binary/test
+    cardano-crypto-class
+    cardano-crypto-praos
+    cardano-crypto-tests
+    measures
+    orphans-deriving-via
+    slotting
+    strict-containers
+
 source-repository-package
   type: git
   location: https://github.com/input-output-hk/plutus
@@ -264,23 +150,3 @@ source-repository-package
     plutus-core
     prettyprinter-configurable
     word-array
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/goblins
-  tag: cde90a2b27f79187ca8310b6549331e59595e7ba
-  --sha256: 17c88rbva3iw82yg9srlxjv2ia5wjb9cyqw44hik565f5v9svnyg
-
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/cardano-crypto
-  tag: f73079303f663e028288f9f4a9e08bcca39a923e
-  --sha256: 1n87i15x54s0cjkh3nsxs4r1x016cdw1fypwmr68936n3xxsjn6q
-
--- Drops an instance breaking the Plutus code. Should be released to Hackage
--- eventually.
-source-repository-package
-  type: git
-  location: https://github.com/input-output-hk/flat
-  tag: ee59880f47ab835dbd73bea0847dab7869fc20d8
-  --sha256: 1lrzknw765pz2j97nvv9ip3l1mcpf2zr4n56hwlz0rk7wq7ls4cm
diff --git a/network-mux/network-mux.cabal b/network-mux/network-mux.cabal
@@ -124,7 +124,7 @@ test-suite test
   build-depends:       base,
                        io-classes,
                        strict-stm,
-                       io-sim           ^>=0.3,
+                       io-sim            >=0.3 && < 0.4,
                        contra-tracer,
                        network-mux,
                        Win32-network,

diff --git a/nix/default.nix b/nix/default.nix
@@ -48,7 +48,7 @@ let
             };
         })
       # And, of course, our haskell-nix-ified cabal project:
-      (import ./pkgs.nix)
+      (import ./pkgs.nix { inherit sources; })
     ] ++ [
       # This overlay adds a field localConfig to the pkgs that will be used
       # afterwards to retrieve the locally defined values for building the