Automatically generate cache for stackage projects

lib/stack-cache-generator.nix

@@ -0,0 +1,34 @@
+# Generate cache entries for dependencies of package defined in `src`
+
+{ pkgs }:
+{ src, stackYaml ? "stack.yaml" }:
+let
+    s2n = import ../pkgs/stack-to-nix/lib.nix pkgs;
+
+    # All repos served via ssh or git protocols are usually private
+    private = url: pkgs.lib.substring 0 4 url != "http";
+
+    deps = (s2n.importYAML "${src}/${stackYaml}").extra-deps or [ ];
+    hashPath = path:
+        builtins.readFile (pkgs.runCommand "hash-path" { preferLocalBuild = true; }
+            "echo -n $(${pkgs.nix}/bin/nix-hash --type sha256 --base32 ${path}) > $out");
+in with pkgs.lib;
+concatMap (dep:
+    if !builtins.isAttrs dep then
+        [ ]
+    else
+        let
+            pkgsrc = builtins.fetchGit {
+                url = dep.git;
+                ref = "*";
+                rev = dep.commit;
+            };
+        in map (subdir:
+            rec {
+                name = s2n.cabalPackageName "${pkgsrc}/${subdir}";
+                rev = dep.commit;
+                url = dep.git;
+                is-private = private url;
+                sha256 = if !is-private then hashPath pkgsrc else null;
+            } // (optionalAttrs (subdir != "") { inherit subdir; }))
+        (dep.subdirs or [ "" ])) deps

overlays/haskell.nix

@@ -298,6 +298,10 @@ self: super: {
               text = self.buildPackages.lib.concatMapStringsSep "\n" mkCacheLine repos;
           };
 
+        genStackCache = import ../lib/stack-cache-generator.nix {
+            inherit (self.buildPackages) pkgs;
+        };
+
         mkCacheModule = cache:
             # for each item in the `cache`, set
             #   packages.$name.src = fetchgit ...
@@ -313,8 +317,6 @@ self: super: {
             # src value.
             #
             # TODO: this should be moved into `call-stack-to-nix`
-            #       it should be automatic and not the burden of
-            #       the end user to work around nix peculiarities.
             { packages =
                 let
                   repoToAttr = { name, url, rev, ref ? null, sha256 ? null, subdir ? null, is-private ? false, ... }: {
@@ -334,6 +336,7 @@ self: super: {
                   cacheMap = builtins.map repoToAttr cache;
                 in
                 builtins.foldl' (x: y: x // y) {} cacheMap;
+
             };
 
         # Takes a haskell src directory runs cabal new-configure and plan-to-nix.
@@ -408,13 +411,24 @@ self: super: {
 
         stackProject' =
             { ... }@args:
-            let stack = importAndFilterProject (callStackToNix args);
+            let stack = importAndFilterProject (callStackToNix ({ inherit cache; } // args));
+                generatedCache = genStackCache {
+                    inherit (args) src;
+                    stackYaml = args.stackYaml or "stack.yaml";
+                };
+                cache = args.cache or (builtins.trace
+                    (builtins.trace ''
+                        Automatically generated cache for this project.
+                        You can pass it as a cache argument to speed up builds:
+                    ''
+                    # Force evaluation so that tracing prints out the whole list
+                    (builtins.deepSeq generatedCache generatedCache)) generatedCache);
             in let pkg-set = mkStackPkgSet
                 { stack-pkgs = stack.pkgs;
                   pkg-def-extras = (args.pkg-def-extras or []);
-                  modules = (args.modules or [])
-                          ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; }
-                          ++ self.lib.optional (args ? cache) (mkCacheModule args.cache);
+                  modules =  self.lib.singleton (mkCacheModule cache)
+                             ++ (args.modules or [])
+                             ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; };
                 };
             in { inherit (pkg-set.config) hsPkgs; stack-nix = stack.nix; };
 

pkgs/stack-to-nix/COPYING

@@ -0,0 +1,20 @@
+Copyright (c) 2018 Eelco Dolstra and the Nixpkgs/NixOS contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pkgs/stack-to-nix/README.md

@@ -0,0 +1,36 @@
+stack-to-nix
+===========
+
+This is a handy nix function for building stack projects with nix, fully inside the nix sandbox.
+It parses the stack.yaml file and translates the dependencies into a series of fetchUrl, fetchGit calls.
+
+Example
+=======
+
+```nix
+{ pkgs }:
+
+let
+  stackToNix = pkgs.callPackage (fetchTarball https://github.com/serokell/stack-to-nix/archive/master.tar.gz) { };
+in
+stackToNix {
+  # root: the path with stack.yaml. you may want to filter this. for example using nix-gitignore.
+  root = ./.;
+  # shell: get the .env instead of the nix-build derivation. recommended that you do this with shell.nix/default.nix.
+  # see https://github.com/DisciplinaOU/disciplina/blob/master/shell.nix
+  shell = false;
+  # you shouldn't need overrides, but you can ;)
+  overrides = final: previous: with pkgs.haskell.lib; {
+    qtah = overrideCabal previous.qtah (super: {
+      libraryToolDepends = with pkgs.qt5; [ qtbase qttools ];
+    });
+  };
+}
+```
+
+Problems
+========
+
+If you use this on a repo with git dependencies,
+you will need [NixOS/nix#2409](https://github.com/NixOS/nix/pull/2409). It's in our patch set. `nix-env -f https://github.com/serokell/serokell-closure/archive/master.tar.gz -iA nix`
+

pkgs/stack-to-nix/build.nix

@@ -0,0 +1,113 @@
+{ overrides, pkgs, shell, stackage }: project: root:
+
+with pkgs;
+with lib;
+
+let
+  inherit (haskell.lib) overrideCabal;
+
+  stackagePackages = (import stackage) stackagePackages pkgs;
+
+  resolverName = replaceStrings ["."] [""] project.resolver;
+  resolver = stackagePackages.haskell.packages.stackage."${resolverName}";
+
+  snapshot = resolver.override {
+    overrides = mergeExtensions [
+      defaultDeps
+      extraDeps
+      localDeps
+      overrides
+    ];
+  };
+
+  inherit (snapshot) callHackage;
+
+  defaultDeps = final: previous: {
+    mkDerivation = drv: previous.mkDerivation (drv // {
+      doCheck = false;
+      doHaddock = false;
+      enableExecutableProfiling = false;
+      enableLibraryProfiling = false;
+    });
+  };
+
+  handlers = import ./handlers.nix pkgs;
+
+  handleExtra = spec:
+    let
+      handler = findFirst (h: h.test spec)
+        (throw "can't handle extra dep: ${spec}") handlers;
+    in
+    handler.handle spec;
+
+  extraSpecs = project.extra-deps or [];
+
+  extraDeps =
+    mergeExtensions (map (spec: final: const (handleExtra spec final)) extraSpecs);
+
+  withStrictDeps = drv: drv.overrideAttrs (const { strictDeps = true; });
+
+  localPackage = name: path:
+    let
+      drv = cabalToNix snapshot name root {} ''--subpath="${path}"'';
+      overrides = const {
+        doBenchmark = true;
+        doCheck = true;
+        doHaddock = true;
+        license = licenses.free;
+      };
+    in
+    withStrictDeps (overrideCabal drv overrides);
+
+  localAttrs = listToAttrs
+    (map (path: nameValuePair (cabalPackageName "${root}/${path}") path) (project.packages or ["."]));
+
+  localDeps = final: previous:
+    mapAttrs localPackage localAttrs;
+
+  target = mapAttrs (name: const (getAttr name snapshot)) localAttrs;
+
+  localPaths = map (removePrefix "./") (project.packages or ["."]);
+
+  stackSnapshot = {
+    inherit (project) resolver;
+    packages = extraSpecs;
+  };
+
+  pathHash = path:
+    builtins.unsafeDiscardStringContext (builtins.substring 0 32 (baseNameOf path));
+
+  stackSnapshotWithName = stackSnapshot // {
+    name = pathHash (exportYAML stackSnapshot);
+  };
+
+  stackConfig = {
+    packages = project.packages or ["."];
+    resolver = exportYAML stackSnapshotWithName;
+  };
+
+  shellEnv = snapshot.shellFor {
+    packages = const (attrValues target);
+    nativeBuildInputs = [ cabal-install stack ];
+
+    STACK_IN_NIX_SHELL = 1;
+    STACK_IN_NIX_EXTRA_ARGS = "";
+    STACK_PLATFORM_VARIANT = "nix";
+    STACK_YAML = "stack-to-nix.yaml";
+
+    shellHook = ''
+      cat ${exportYAML stackConfig} > stack-to-nix.yaml
+
+      for f in $(find * -name package.yaml); do
+        ${snapshot.hpack}/bin/hpack --force $f
+      done
+
+      echo packages: > cabal.project
+      for spec in ${concatStringsSep " " localPaths}; do
+        echo "  $spec" >> cabal.project
+      done
+    '';
+  };
+in
+
+if shell then shellEnv else target

pkgs/stack-to-nix/cabal-name/cabal-name.cabal

@@ -0,0 +1,9 @@
+name: cabal-name
+version: 1
+cabal-version: >= 1.2
+
+executable cabal-name
+  build-depends: base, Cabal
+  default-language: Haskell2010
+  hs-source-dirs: src
+  main-is: Main.hs

pkgs/stack-to-nix/cabal-name/default.nix

@@ -0,0 +1,19 @@
+{ pkgs ? import <nixpkgs> {} }: with pkgs;
+
+let
+  project = { mkDerivation, base, Cabal }: mkDerivation {
+    pname = "cabal-name";
+    version = "1";
+
+    src = lib.cleanSource ./.;
+
+    isExecutable = true;
+    isLibrary = false;
+
+    executableHaskellDepends = [ base Cabal ];
+
+    license = lib.licenses.free;
+  };
+in
+
+haskellPackages.callPackage project {}

pkgs/stack-to-nix/cabal-name/src/Main.hs

@@ -0,0 +1,14 @@
+module Main where
+
+import Distribution.PackageDescription.Parsec
+import Distribution.Types.GenericPackageDescription
+import Distribution.Types.PackageDescription
+import Distribution.Types.PackageId
+import Distribution.Types.PackageName
+import Distribution.Verbosity
+import System.Environment
+
+main = do
+  args <- getArgs
+  desc <- readGenericPackageDescription normal (head args)
+  print $ (unPackageName . pkgName . package . packageDescription) desc

pkgs/stack-to-nix/default.nix

@@ -0,0 +1,17 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+{ overrides ? (_: _: {})
+, root
+, shell ? false
+, stackage ? fetchGit { url = "https://github.com/typeable/nixpkgs-stackage"; rev = "6042df5e646d65b826add0a85d16304bee8e1dd5"; } }:
+
+let
+  lib = import ./lib.nix pkgs;
+
+  buildProject = import ./build.nix {
+    pkgs = pkgs // { lib = pkgs.lib // lib; };
+    inherit overrides shell stackage;
+  };
+in
+
+buildProject (lib.importYAML "${root}/stack.yaml") root

pkgs/stack-to-nix/handlers.nix

@@ -0,0 +1,36 @@
+pkgs: with pkgs; with lib;
+
+[
+  ({
+    handle = spec: self:
+      let
+        components = splitString "-" spec;
+        name = concatStringsSep "-" (init components);
+        version = last components;
+      in
+      { "${name}" = lib.callHackage self name version {}; };
+
+    test = isString;
+  })
+  ({
+    handle = spec: self:
+      let
+        src = fetchGit {
+          url = spec.git;
+          ref = "*";
+          rev = spec.commit;
+        };
+
+        subdirToAttr = subdir:
+          let
+            name = cabalPackageName "${src}/${subdir}";
+          in
+          nameValuePair name (cabalToNix self name src {} ''--subpath="${subdir}"'');
+
+        subdirs = spec.subdirs or [ "." ];
+      in
+      listToAttrs (map subdirToAttr subdirs);
+
+    test = spec: isAttrs spec && spec ? git;
+  })
+]