Overlays (#261)

The Overlays branch

This is a major reorganization in how haskell.nix is used, and marks our 1.0 release.
The branch doesn't build due to numerous issues that we believe to be with the CI
and not this branch. We expect only very minor adjustments prior to calling this the
official 1.0 release.

* Move iohk-nix patches into haskell.nix

This moves the customizations we did in iohk-nix
into haskell.nix via overlays and config.

Add bootPkgs logic

this moves the nuking of bootPkgs where it belongs. This should eventually still be
removed and replaced by a proper solution, that doesn't require the nuking of
bootPkgs.

Allow us to bootstrap binary ghcs and a cabal-install

With this we can do the following:

```
$ nix repl compiler/old-ghc-nix
nix-repl> :b (let pkgs = import <nixpkgs> {}; in with import ./. {}; import ./compiler/bootstrap/cabal-install.nix { inherit (pkgs) fetchurl stdenv zlib; inherit hackage
; ghc = ghc844; src = pkgs.fetchurl { url = "https://github.com/haskell/cabal/archive/Cabal-v3.0.0.0-rc3.tar.gz"; sha256 = "1zl2mgg8307ykq3v8nmafc6zdhhj1cw7w8ffpap16dsm6
5lbnx33"; }; })
```

which wile it may look daunting, will allow us to bootstrap a
cabal-install with a ghc. From that point onwards, we should be able to
build any hackage package via haskell.nix.

Pass through cabal-install version

Better threading of arguments.

Add bootstrap overlay

Allow alex + happy to be built

This still has a wart: we need nix-tools, and for that we use the ghc865 from
nixpkgs. Ideally we'd build nix-tools against a specific ghc, but then
we'd need a build expression for that.

Make ghcjs work

Building something like this:
```
 nix build '(with import ./. { nixpkgs = ../nixpkgs; nixpkgsArgs = { crossSystem = { config = "js-unknown-ghcjs"; }; }; }; (haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; })).components.exes.hello'
```
will now work. Assuming `nixpkgs` has been appropriately patched to support the `js-unknown-ghcjs` triple.

Also: this will need an additional `Cabal` patch, to make `Cabal` understand what it needs to do with: `dist/build/hello/hello: copyFile: does not exist (No such file or directory)`

It needs to learn that `dist/build/hello/hello.jsexe` is what it wants to copy and that that is a directory.  Luckily we do have some code in Cabal already that does this for `.exe` on windows.

Build `js-unknown-ghcjs` packages with haskell.nix

Using the following expression:
```
with import ./. { nixpkgs = ../nixpkgs; nixpkgsArgs = { crossSystem = { config = "js-unknown-ghcjs"; }; }; };
let Cabal = buildPackages.haskell-nix.hackage-package {
    name = "Cabal"; version = "2.4.1.0";
    modules = [
        { packages.Cabal.patches = [ ./Cabal-install-folder.diff ]; }
    ];
}; in
(haskell-nix.hackage-package {
    name = "hello"; version = "1.0.0.2";
    modules = [
    	    ({config, ... }:{ packages.hello.package.setup-depends = [ Cabal ]; })
    ];}).components.exes.hello
```
in a `test.nix` file. And running
```
nix build -f ./test.nix
```
on it, will produce
```
./result
├── bin
│   └── hello.jsexe
│       ├── all.js
│       ├── all.js.externs
│       ├── index.html
│       ├── lib.js
│       ├── manifest.webapp
│       ├── out.frefs.js
│       ├── out.frefs.json
│       ├── out.js
│       ├── out.stats
│       ├── rts.js
│       └── runmain.js
└── share
    └── doc
        └── x86_64-linux-ghc-8.6.5
            └── hello-1.0.0.2
                └── LICENSE

6 directories, 12 files
```

Author: angerman

.gitattributes

@@ -2,5 +2,5 @@
 # Hide stuff that is generated by nix-tools plan-to-nix
 
 nix-tools/.plan.nix/*.nix linguist-generated=true
-nix-tools/.plan-pkgs.nix linguist-generated=true
+nix-tools/pkgs.nix linguist-generated=true
 .stack-to-nix.cache linguist-generated=true

README.org

@@ -8,18 +8,53 @@
 =haskell.nix= is an experimental new builder for Haskell packages.
 
 It works by automatically translating your Cabal or Stack project and
-its dependencies into Nix code. Most of your dependencies are already
-translated, so you generally won't have too much generated code.
+its dependencies into Nix code.  It provides IFD (imports from derviation)
+functions that can minimize the amount of nix code you need to add.
 
 For the documentation, see https://input-output-hk.github.io/haskell.nix/.
 
 ** Quickstart
 
-This will download and build =nix-tools=.
+For =cabal.project= project add a =default.nix=:
 
 #+begin_src sh
-nix build -f https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz nix-tools -o nt
-./nt/bin/stack-to-nix
+{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
+, haskellCompiler ? "ghc865"
+}:
+  pkgs.haskell-nix.cabalProject {
+    src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; };
+    ghc = pkgs.buildPackages.pkgs.haskell.compiler.${haskellCompiler};
+  }
+#+end_src
+
+For a =stack.yaml= project add a =default.nix=:
+
+#+begin_src sh
+{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
+}:
+  pkgs.haskell-nix.stackProject {
+    src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; };
+  }
+#+end_src
+
+To build the library component of a package in the project run:
+
+#+begin_src sh
+nix build -f . your-package-name.components.library
+#+end_src
+
+To build an executable:
+
+#+begin_src sh
+nix build -f . your-package-name.components.exes.your-exe-name
+#+end_src
+
+To open a shell for use with `cabal` run:
+
+#+begin_src sh
+nix-shell -A shellFor
+cabal new-build your-package-name
+cabal new-repl your-package-name:library:your-package-name
 #+end_src
 
 ** Related repos

build.nix

@@ -3,21 +3,32 @@
 # It is separate from default.nix because that file is the public API
 # of Haskell.nix, which shouldn't have tests, etc.
 
-{ pkgs ? import nixpkgs nixpkgsArgs
-, nixpkgs ? ./nixpkgs
+{ nixpkgs ? ./nixpkgs
 # Provide args to the nixpkgs instantiation.
 , system ? builtins.currentSystem
 , crossSystem ? null
 , config ? {}
-, nixpkgsArgs ? { inherit system crossSystem config; }
+, nixpkgsArgs ? { inherit system crossSystem; }
 }:
 
 let
-  haskell = import ./default.nix { inherit pkgs; };
+  haskellNixArgs = import ./default.nix;
+  pkgs = import nixpkgs ({
+    config   = haskellNixArgs.config // config;
+    overlays = haskellNixArgs.overlays ++
+      [(self: super: {
+        darcs = (self.haskell-nix.hackage-package {
+          name = "darcs";
+          version = "2.14.2";
+          # Apply the latest darcs.net Setup.hs patches
+          modules = [{packages.darcs.patches = [ ./patches/darcs-setup.patch ];}];
+        }).components.exes.darcs;
+      })]; } // nixpkgsArgs);
+  haskell = pkgs.haskell-nix;
 
 in {
   inherit (haskell) nix-tools source-pins;
-  tests = import ./test/default.nix { inherit haskell; };
+  tests = import ./test/default.nix { inherit nixpkgs nixpkgsArgs; };
 
   # Scripts for keeping Hackage and Stackage up to date, and CI tasks.
   maintainer-scripts = pkgs.dontRecurseIntoAttrs {

builder/comp-builder.nix

@@ -191,7 +191,7 @@ stdenv.mkDerivation ({
   buildPhase = ''
     runHook preBuild
     # https://gitlab.haskell.org/ghc/ghc/issues/9221
-    $SETUP_HS build -j$(($NIX_BUILD_CORES > 4 ? 4 : $NIX_BUILD_CORES)) ${lib.concatStringsSep " " (component.setupBuildFlags ++ setupGhcOptions)}
+    $SETUP_HS build ${haskellLib.componentTarget componentId} -j$(($NIX_BUILD_CORES > 4 ? 4 : $NIX_BUILD_CORES)) ${lib.concatStringsSep " " (component.setupBuildFlags ++ setupGhcOptions)}
     runHook postBuild
   '';
 

builder/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, buildGHC, fetchurl, pkgconfig, nonReinstallablePkgs, hsPkgs }:
+{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, fetchurl, pkgconfig, nonReinstallablePkgs, hsPkgs }:
 
 let
   # Builds a single component of a package.
@@ -7,7 +7,7 @@ let
   };
 
   setup-builder = haskellLib.weakCallPackage pkgs ./setup-builder.nix {
-    ghc = buildGHC;
+    ghc = (ghc.passthru.buildGHC or ghc);
     hsPkgs = hsPkgs.buildPackages;
     inherit haskellLib nonReinstallablePkgs makeSetupConfigFiles;
   };
@@ -28,7 +28,7 @@ let
   # When building setup depends we need to use the build systems GHC and Packages
   makeSetupConfigFiles = haskellLib.weakCallPackage buildPackages ./make-config-files.nix {
     inherit haskellLib nonReinstallablePkgs;
-    ghc = buildGHC;
+    ghc = (ghc.passthru.buildGHC or ghc);
   };
 
 
@@ -57,7 +57,7 @@ in {
   # Build a Haskell package from its config.
   # TODO: this pkgs is the adjusted pkgs, but pkgs.pkgs is unadjusted
   build-package = haskellLib.weakCallPackage pkgs ./hspkg-builder.nix {
-    inherit haskellLib ghc buildGHC comp-builder setup-builder;
+    inherit haskellLib ghc comp-builder setup-builder;
   };
 
   inherit shellFor;

builder/ghc-for-component-wrapper.nix

@@ -40,6 +40,10 @@ in runCommand "${componentName}-${ghc.name}" {
     rm -rf ${libDir}/*/
     # ... but retain the lib/ghc/bin directory. This contains `unlit' and friends.
     ln -s ${ghc}/lib/${ghcCommand}-${ghc.version}/bin ${libDir}
+    # ... and the ghcjs shim's if they are available
+    if [ -d ${ghc}/lib/${ghcCommand}-${ghc.version}/shims ]; then
+      ln -s ${ghc}/lib/${ghcCommand}-${ghc.version}/shims ${libDir}
+    fi
     # Replace the package database with the one from target package config.
     ln -s ${configFiles}/package.conf.d ${packageCfgDir}
 

builder/hspkg-builder.nix

@@ -1,4 +1,4 @@
-{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, buildGHC, fetchurl, runCommand, comp-builder, setup-builder }:
+{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, fetchurl, runCommand, comp-builder, setup-builder }:
 
 
 { flags
@@ -30,17 +30,17 @@ let
     import Distribution.Simple
     main = defaultMain
   '';
-  defaultSetup = buildPackages.runCommand "default-Setup" { nativeBuildInputs = [buildGHC]; } ''
+  defaultSetup = buildPackages.runCommand "default-Setup" { nativeBuildInputs = [(ghc.passthru.buildGHC or ghc)]; } ''
     cat ${defaultSetupSrc} > Setup.hs
     mkdir -p $out/bin
-    ${buildGHC.targetPrefix}ghc Setup.hs --make -o $out/bin/Setup
+    ${(ghc.passthru.buildGHC or ghc).targetPrefix}ghc Setup.hs --make -o $out/bin/Setup
   '';
 
-  setup = if package.buildType == "Simple"
+  setup = if package.buildType == "Simple" && package.setup-depends == []
     then defaultSetup
     else setup-builder {
       setup-depends = package.setup-depends;
-      inherit package name src flags;
+      inherit package name src flags revision patches defaultSetupSrc;
     };
 
   buildComp = componentId: component: comp-builder {

builder/make-config-files.nix

@@ -28,7 +28,7 @@ let
       # so we are dealing with a sublib. As we build sublibs separately, the above
       # query should be safe.
       echo "--dependency=''${name#z-${p}-z-}=$id" >> $out/configure-flags
-    fi     
+    fi
     if ver=$(target-pkg ${pdbArg} field ${p} version --simple-output); then
       echo "constraint: ${p} == $ver" >> $out/cabal.config
       echo "constraint: ${p} installed" >> $out/cabal.config
@@ -105,6 +105,9 @@ in { identifier, component, fullName, flags ? {} }:
   #
   # TODO: Fix Cabal.
   # TODO: this is only needed if we do dynamic libraries.
+  #
+  # NOTE [ln -s -f]: we force link, as we may have dependencies that contain shared deps
+  #                  (e.g. libiconv), and thus we don't want to fail, but just link it again.
   + lib.optionalString stdenv.isDarwin ''
     # Work around a limit in the macOS Sierra linker on the number of paths
     # referenced by any one dynamic library:
@@ -114,7 +117,7 @@ in { identifier, component, fullName, flags ? {} }:
     local dynamicLinksDir="$out/lib/links"
     mkdir -p $dynamicLinksDir
     for d in $(grep dynamic-library-dirs "$out/package.conf.d/"*|awk '{print $2}'|sort -u); do
-      ln -s "$d/"*.dylib $dynamicLinksDir
+      ln -f -s "$d/"*.dylib $dynamicLinksDir
     done
     # Edit the local package DB to reference the links directory.
     for f in "$out/package.conf.d/"*.conf; do

builder/setup-builder.nix

@@ -1,6 +1,6 @@
 { stdenv, lib, buildPackages, haskellLib, ghc, nonReinstallablePkgs, hsPkgs, makeSetupConfigFiles }:
 
-{ setup-depends, package, name, src, flags }:
+{ setup-depends, package, name, src, flags, revision, patches, defaultSetupSrc }:
 
 let
   component = {
@@ -34,7 +34,7 @@ let
 
 in
  stdenv.lib.fix (drv:
-    stdenv.mkDerivation {
+    stdenv.mkDerivation ({
       name = "${fullName}";
       src = cleanSrc;
       nativeBuildInputs = [ghc];
@@ -45,8 +45,11 @@ in
         inherit configFiles cleanSrc;
       };
 
-      phases = ["unpackPhase" "buildPhase" "installPhase"];
+      phases = ["unpackPhase" "patchPhase" "buildPhase" "installPhase"];
       buildPhase = ''
+        if [[ ! -f ./Setup.hs  && ! -f ./Setup.lhs ]]; then
+          cat ${defaultSetupSrc} > Setup.hs
+        fi
         for f in Setup.hs Setup.lhs; do
           if [ -f $f ]; then
             echo Compiling package $f
@@ -62,4 +65,6 @@ in
         mkdir -p $out/bin
         install ./Setup $out/bin/Setup
       '';
-    })
+    }
+    // lib.optionalAttrs (patches != []) { patches = map (p: if builtins.isFunction p then p { inherit (package.identifier) version; inherit revision; } else p) patches; }
+  ))