Skip to content

Add support for Nix flakes and haskell.nix #1570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 56 commits into from
Apr 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
a3a8d73
Initial version of flake.nix
jwiegley Nov 30, 2022
b87fe81
Further work on flake.nix
jwiegley Nov 30, 2022
ea46017
Update flake.nix and cabal.project
jwiegley Nov 30, 2022
3bd2bdc
Add old Nix files back, they don't conflict
jwiegley Dec 1, 2022
c8089b0
Merge branch 'master' into johnw/nix-flake
jwiegley Dec 5, 2022
7292803
Merge branch 'master' into johnw/nix-flake
jwiegley Dec 6, 2022
5d9695e
Merge branch 'master' into johnw/nix-flake
jwiegley Feb 23, 2023
5161f61
Update the cabal.project hashes
enobayram Apr 13, 2023
75030b2
Align default.nix with flake.nix
enobayram Apr 13, 2023
3d154e6
Merge branch 'master' into johnw/nix-flake
enobayram Apr 13, 2023
bb5b8ab
Avoid the haddock error from http2
enobayram Apr 20, 2023
0720e79
Merge branch 'master' into johnw/nix-flake
jwiegley Apr 24, 2023
1dd99bd
Pin explicit commit of nixpkgs
enobayram Apr 25, 2023
024569f
Add a GitHub action for cached nix builds
enobayram Apr 25, 2023
86855e2
Point "on push" branch to master
enobayram Apr 25, 2023
3db800f
Run on all new commits
enobayram Apr 25, 2023
753fadd
Convert tabs to spaces in default.nix
enobayram Apr 25, 2023
4f2cce7
Rename Nix github action
enobayram Apr 25, 2023
6f6e3b7
Add a workflow_dispatch trigger
enobayram Apr 25, 2023
ce0beee
Try a different way to set the trigger
enobayram Apr 25, 2023
726e663
Revert "Try a different way to set the trigger"
enobayram Apr 25, 2023
921085d
Make sure default.nix sees same src as flake.nix
enobayram Apr 25, 2023
f1c0491
Update nix.yml
DevopsGoth Apr 25, 2023
e96160e
Update nix.yml
DevopsGoth Apr 25, 2023
33c6ac0
Update nix.yml
DevopsGoth Apr 25, 2023
e9646b3
increase timeout for nix build
DevopsGoth Apr 26, 2023
4922da9
Fix the nix develop invocation
enobayram Apr 26, 2023
4539616
Update action versions to avoid warnings
enobayram Apr 26, 2023
e559343
Merge remote-tracking branch 'origin/master' into johnw/nix-flake
enobayram Apr 26, 2023
3fb5746
Avoid dowloading the entire nix develop closure
enobayram Apr 26, 2023
c27c482
Avoid depending on the chainweb-node closure too
enobayram Apr 26, 2023
ef3de89
Filter out irrelevant folders in project' src
enobayram Apr 26, 2023
1112bab
Fix Nix issues
enobayram Apr 26, 2023
0dad325
Clean up Nix files
enobayram Apr 26, 2023
5865020
Prefetch haskell.nix IFD closure
enobayram Apr 26, 2023
78801a1
Revert "Prefetch haskell.nix IFD closure"
enobayram Apr 26, 2023
54961bb
Revert "Clean up Nix files"
enobayram Apr 26, 2023
f681a83
Rename default.nix to default-flake.nix
enobayram Apr 26, 2023
6d6d927
Restore the old default.nix
enobayram Apr 26, 2023
fbdbb33
Add note to cabal.project on how to determine the sha values
jwiegley Apr 27, 2023
e170492
Convert cabal.project sha256s to base32
enobayram Apr 28, 2023
1a00b59
Check the cabal.project hashes during nix build
enobayram Apr 28, 2023
4712fde
Move the cabal.project check outside of build
enobayram Apr 28, 2023
eca7552
Separate the executable and devShell checks
enobayram Apr 28, 2023
4281ac9
Add jq to check-cabal-project PATH
enobayram Apr 28, 2023
8f5d288
Move check-cabal-project to default-flake.nix
enobayram Apr 28, 2023
eb31863
Fake commit to test cabal.project check
enobayram Apr 28, 2023
c5d4d12
Revert "Fake commit to test cabal.project check"
enobayram Apr 28, 2023
ef7e468
Expose the haskell source folder derivation
enobayram Apr 28, 2023
043c25a
Remove the cabal.project check
enobayram Apr 28, 2023
6e124e4
Revert "Convert cabal.project sha256s to base32"
enobayram Apr 28, 2023
2666565
Fake commit to test outdated sha256
enobayram Apr 28, 2023
1921ca2
Revert "Fake commit to test outdated sha256"
enobayram Apr 28, 2023
ae32d93
Fix and improve the source filtering
enobayram Apr 28, 2023
9a2cd1c
Fake commit to test source filtering
enobayram Apr 28, 2023
d9634a6
Revert "Fake commit to test source filtering"
enobayram Apr 28, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion .github/workflows/nix.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,31 @@
name: Build and cache with Nix

on:
workflow_dispatch:
workflow_dispatch:
push:

jobs:
build-and-cache:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Set up Nix with caching
uses: kadena-io/setup-nix-with-cache@v1
with:
cache_url: s3://nixcache.chainweb.com?region=us-east-1
signing_private_key: ${{ secrets.NIX_CACHE_PRIVATE_KEY }}

- name: Set up AWS credentials
uses: aws-actions/configure-aws-credentials@v2
with:
aws-access-key-id: ${{ secrets.NIX_CACHE_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.NIX_CACHE_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1

- name: Build and cache artifacts
timeout-minutes: 740
run: |
echo Building the project and its devShell
nix build .#check
11 changes: 11 additions & 0 deletions cabal.project
Original file line number Diff line number Diff line change
Expand Up @@ -50,31 +50,41 @@ package yet-another-logger

-- -------------------------------------------------------------------------- --
-- Source Repository Packages
--
-- In order to determine proper sha256 value each time the revision is
-- changed, please run the following command:
--
-- nix-prefetch-git --url <location> --rev <tag>

source-repository-package
type: git
location: https://github.com/kadena-io/pact.git
tag: 842fbc4256b3cbbde337dbeaa393b649a26f1574
--sha256: sha256-jD4Y2NOSYT0EQHs5BRSVvu8asfvRwt5XLDuZwbcG4mk=
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How are those pins determined when updating the source tag for a package? Is that somewhere documented?

In the past we would just wait for the zeus nix build to fail and copy and paste from the failure message? That wasn't ideal (because of the extra CI roundtrip) but worked. Will that method still work?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any mechanism that guarantees that the source version represented by the --sha value matches the sources of the source tag?

In the past (with the old nix infra) we ran into issues where nix pins didn't exactly match the sources of the respective Hackage version and there was no easy way to detected that.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The command to determine the sha locally is:

nix-prefetch-git --url https://github.com/kadena-io/pact.git --rev <SHA>

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add that as a comment to the cabal.project file? So when people update that file they find that info?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we include an action in the CI workflow that runs that command to double checks that the nix --sha actually matches the source tag?

(You may take a look at cabal CI workflows; there are examples for actions that check that the git tree is still clean after performing some action or test.)

Copy link
Contributor

@enobayram enobayram Apr 28, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@larskuhtz Earlier today, I've added a step to check the hashes during the github action build, but then later I realized that the check was redundant, because it doesn't seem to be possible to start the build with a wrong sha256 anyway. I've tried multiple ways to fool it, like changing the source-repository-package commit hash without updating the sha256, but the nix build step always failed quickly, telling the correct sha256. As an example, check out this run: https://github.com/kadena-io/chainweb-node/actions/runs/4831581150/jobs/8609217888

The build took 2 minutes to fail and the error message tells you how you need to update the hash:

error: hash mismatch in fixed-output derivation '/nix/store/h9wwd3i3g0shagpc50i5kkms0mbbm5lc-pact-baac04e.drv':
         specified: sha256-jD4Y2NOSYT0EQHs5BRSVvu8asfvRwt5XLDuZwbcG4mk=
            got:    sha256-PmvS1kNvnOErMpfQPhGp4oOxuZLa34MEXt/Q1p09EKI=

In the past (with the old nix infra) we ran into issues where nix pins didn't exactly match the sources of the respective Hackage version and there was no easy way to detected that.

I can imagine how the old nix infra running on Zeus could perform misleading builds, picking the old source folder from the nix store due to the outdated hash, but I don't think that kind of inconsistency is possible with this new setup.

That's why I've removed the separate cabal.project checks.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Admittedly, the error message you get from the Nix build is not ideal because it doesn't tell you what exactly you need to do. So if somebody encounters this message out of the blue, it could be frustrating to try to debug it.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could add a hint about how to read this error to the comment that describes how to compute the sha.


source-repository-package
type: git
location: https://github.com/kadena-io/chainweb-storage.git
tag: af9f42c78bcd703ac382cfd8101c6d1c66a035f1
--sha256: sha256-+C4yNKjf0SY3mwxjmHJ8OJGAfcZz1HTFQyT09fhQ9Co=

source-repository-package
type: git
location: https://github.com/kadena-io/rocksdb-haskell.git
tag: c2b3dd8bb714a12ea6763565d168a03df38fcc58
--sha256: sha256-gLkxfHeg4D7S53Uzt807Q1bD0/s5dO7b+51Rbrq2XYg=

source-repository-package
type: git
location: https://github.com/kadena-io/rosetta.git
tag: 6c8dd2eea1f6d0dba925646dbcb6e07feeccbfd5
--sha256: sha256-EE8A/5wJYqIfx1uz/v3QjWCFgaeVLf/fi1r03A3w8qY=

source-repository-package
type: git
location: https://github.com/kadena-io/kadena-ethereum-bridge.git
tag: 10f21e96af1dce4f13e261be9dfad8c28cd299f7
--sha256: sha256-E+9yLZYLcRd4PoBeQXWhOS+ddpVO6jDP6ca+eEwVS+0=

-- Required for GHC-9:

Expand All @@ -83,6 +93,7 @@ source-repository-package
type: git
location: https://github.com/larskuhtz/ixset-typed
tag: d8019c3404d6f3b3c0b0416e9899cfdf614ef425
--sha256: sha256-AwS/uqzUQrTUgxEHNjpX65JKFcaOuZN+03mI16JXuiY=

-- -------------------------------------------------------------------------- --
-- Relaxed Bounds
Expand Down
50 changes: 50 additions & 0 deletions default-flake.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
let flakeDefaultNix = (import (
fetchTarball {
url = "https://github.com/edolstra/flake-compat/archive/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9.tar.gz";
sha256 = "1prd9b1xx8c0sfwnyzkspplh30m613j42l1k789s521f4kv4c2z2"; }
) {
src = ./.;
}).defaultNix;
inputs = flakeDefaultNix.inputs;
pkgsDef = import inputs.nixpkgs (import inputs.haskellNix {}).nixpkgsArgs;
in
{ pkgs ? pkgsDef
, compiler ? "ghc8107"
, flakePath ? flakeDefaultNix.outPath
, nix-filter ? inputs.nix-filter
, ...
}:
let haskellSrc = with nix-filter.lib; filter {
root = flakePath;
exclude = [
".github"
".gitignore"
".gitattributes"
"docs"
"examples"
(matchExt "nix")
"flake.lock"
];
};
chainweb-node = pkgs.haskell-nix.project' {
src = haskellSrc;
compiler-nix-name = compiler;
projectFileName = "cabal.project";
shell.tools = {
cabal = {};
};
shell.buildInputs = with pkgs; [
zlib
pkgconfig
];
modules = [
{
packages.http2.doHaddock = false;
}
];
};
flake = chainweb-node.flake {};
default = flake.packages."chainweb:exe:chainweb-node";
in {
inherit flake default haskellSrc;
}
2 changes: 1 addition & 1 deletion default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -135,4 +135,4 @@ pkgs.haskell.packages.${compiler}.developPackage {
});

inherit returnShellEnv;
}
}
Loading