diff --git a/lib/stack-cache-generator.nix b/lib/stack-cache-generator.nix new file mode 100644 index 0000000000..8bac9d39a4 --- /dev/null +++ b/lib/stack-cache-generator.nix @@ -0,0 +1,34 @@ +# Generate cache entries for dependencies of package defined in `src` + +{ pkgs }: +{ src, stackYaml ? "stack.yaml" }: +let + s2n = import ../pkgs/stack-to-nix/lib.nix pkgs; + + # All repos served via ssh or git protocols are usually private + private = url: pkgs.lib.substring 0 4 url != "http"; + + deps = (s2n.importYAML "${src}/${stackYaml}").extra-deps or [ ]; + hashPath = path: + builtins.readFile (pkgs.runCommand "hash-path" { preferLocalBuild = true; } + "echo -n $(${pkgs.nix}/bin/nix-hash --type sha256 --base32 ${path}) > $out"); +in with pkgs.lib; +concatMap (dep: + if !builtins.isAttrs dep then + [ ] + else + let + pkgsrc = builtins.fetchGit { + url = dep.git; + ref = "*"; + rev = dep.commit; + }; + in map (subdir: + rec { + name = s2n.cabalPackageName "${pkgsrc}/${subdir}"; + rev = dep.commit; + url = dep.git; + is-private = private url; + sha256 = if !is-private then hashPath pkgsrc else null; + } // (optionalAttrs (subdir != "") { inherit subdir; })) + (dep.subdirs or [ "" ])) deps diff --git a/overlays/haskell.nix b/overlays/haskell.nix index 5156ec35ba..694a5b91bd 100644 --- a/overlays/haskell.nix +++ b/overlays/haskell.nix @@ -298,6 +298,10 @@ self: super: { text = self.buildPackages.lib.concatMapStringsSep "\n" mkCacheLine repos; }; + genStackCache = import ../lib/stack-cache-generator.nix { + inherit (self.buildPackages) pkgs; + }; + mkCacheModule = cache: # for each item in the `cache`, set # packages.$name.src = fetchgit ... @@ -313,8 +317,6 @@ self: super: { # src value. # # TODO: this should be moved into `call-stack-to-nix` - # it should be automatic and not the burden of - # the end user to work around nix peculiarities. { packages = let repoToAttr = { name, url, rev, ref ? null, sha256 ? null, subdir ? null, is-private ? false, ... }: { @@ -334,6 +336,7 @@ self: super: { cacheMap = builtins.map repoToAttr cache; in builtins.foldl' (x: y: x // y) {} cacheMap; + }; # Takes a haskell src directory runs cabal new-configure and plan-to-nix. @@ -408,13 +411,24 @@ self: super: { stackProject' = { ... }@args: - let stack = importAndFilterProject (callStackToNix args); + let stack = importAndFilterProject (callStackToNix ({ inherit cache; } // args)); + generatedCache = genStackCache { + inherit (args) src; + stackYaml = args.stackYaml or "stack.yaml"; + }; + cache = args.cache or (builtins.trace + (builtins.trace '' + Automatically generated cache for this project. + You can pass it as a cache argument to speed up builds: + '' + # Force evaluation so that tracing prints out the whole list + (builtins.deepSeq generatedCache generatedCache)) generatedCache); in let pkg-set = mkStackPkgSet { stack-pkgs = stack.pkgs; pkg-def-extras = (args.pkg-def-extras or []); - modules = (args.modules or []) - ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; } - ++ self.lib.optional (args ? cache) (mkCacheModule args.cache); + modules = self.lib.singleton (mkCacheModule cache) + ++ (args.modules or []) + ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; }; }; in { inherit (pkg-set.config) hsPkgs; stack-nix = stack.nix; }; diff --git a/pkgs/stack-to-nix/COPYING b/pkgs/stack-to-nix/COPYING new file mode 100644 index 0000000000..3eedeb2b73 --- /dev/null +++ b/pkgs/stack-to-nix/COPYING @@ -0,0 +1,20 @@ +Copyright (c) 2018 Eelco Dolstra and the Nixpkgs/NixOS contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pkgs/stack-to-nix/README.md b/pkgs/stack-to-nix/README.md new file mode 100644 index 0000000000..d79f4efa60 --- /dev/null +++ b/pkgs/stack-to-nix/README.md @@ -0,0 +1,36 @@ +stack-to-nix +=========== + +This is a handy nix function for building stack projects with nix, fully inside the nix sandbox. +It parses the stack.yaml file and translates the dependencies into a series of fetchUrl, fetchGit calls. + +Example +======= + +```nix +{ pkgs }: + +let + stackToNix = pkgs.callPackage (fetchTarball https://github.com/serokell/stack-to-nix/archive/master.tar.gz) { }; +in +stackToNix { + # root: the path with stack.yaml. you may want to filter this. for example using nix-gitignore. + root = ./.; + # shell: get the .env instead of the nix-build derivation. recommended that you do this with shell.nix/default.nix. + # see https://github.com/DisciplinaOU/disciplina/blob/master/shell.nix + shell = false; + # you shouldn't need overrides, but you can ;) + overrides = final: previous: with pkgs.haskell.lib; { + qtah = overrideCabal previous.qtah (super: { + libraryToolDepends = with pkgs.qt5; [ qtbase qttools ]; + }); + }; +} +``` + +Problems +======== + +If you use this on a repo with git dependencies, +you will need [NixOS/nix#2409](https://github.com/NixOS/nix/pull/2409). It's in our patch set. `nix-env -f https://github.com/serokell/serokell-closure/archive/master.tar.gz -iA nix` + diff --git a/pkgs/stack-to-nix/build.nix b/pkgs/stack-to-nix/build.nix new file mode 100644 index 0000000000..79f8409b1c --- /dev/null +++ b/pkgs/stack-to-nix/build.nix @@ -0,0 +1,113 @@ +{ overrides, pkgs, shell, stackage }: project: root: + +with pkgs; +with lib; + +let + inherit (haskell.lib) overrideCabal; + + stackagePackages = (import stackage) stackagePackages pkgs; + + resolverName = replaceStrings ["."] [""] project.resolver; + resolver = stackagePackages.haskell.packages.stackage."${resolverName}"; + + snapshot = resolver.override { + overrides = mergeExtensions [ + defaultDeps + extraDeps + localDeps + overrides + ]; + }; + + inherit (snapshot) callHackage; + + defaultDeps = final: previous: { + mkDerivation = drv: previous.mkDerivation (drv // { + doCheck = false; + doHaddock = false; + enableExecutableProfiling = false; + enableLibraryProfiling = false; + }); + }; + + handlers = import ./handlers.nix pkgs; + + handleExtra = spec: + let + handler = findFirst (h: h.test spec) + (throw "can't handle extra dep: ${spec}") handlers; + in + handler.handle spec; + + extraSpecs = project.extra-deps or []; + + extraDeps = + mergeExtensions (map (spec: final: const (handleExtra spec final)) extraSpecs); + + withStrictDeps = drv: drv.overrideAttrs (const { strictDeps = true; }); + + localPackage = name: path: + let + drv = cabalToNix snapshot name root {} ''--subpath="${path}"''; + overrides = const { + doBenchmark = true; + doCheck = true; + doHaddock = true; + license = licenses.free; + }; + in + withStrictDeps (overrideCabal drv overrides); + + localAttrs = listToAttrs + (map (path: nameValuePair (cabalPackageName "${root}/${path}") path) (project.packages or ["."])); + + localDeps = final: previous: + mapAttrs localPackage localAttrs; + + target = mapAttrs (name: const (getAttr name snapshot)) localAttrs; + + localPaths = map (removePrefix "./") (project.packages or ["."]); + + stackSnapshot = { + inherit (project) resolver; + packages = extraSpecs; + }; + + pathHash = path: + builtins.unsafeDiscardStringContext (builtins.substring 0 32 (baseNameOf path)); + + stackSnapshotWithName = stackSnapshot // { + name = pathHash (exportYAML stackSnapshot); + }; + + stackConfig = { + packages = project.packages or ["."]; + resolver = exportYAML stackSnapshotWithName; + }; + + shellEnv = snapshot.shellFor { + packages = const (attrValues target); + nativeBuildInputs = [ cabal-install stack ]; + + STACK_IN_NIX_SHELL = 1; + STACK_IN_NIX_EXTRA_ARGS = ""; + STACK_PLATFORM_VARIANT = "nix"; + STACK_YAML = "stack-to-nix.yaml"; + + shellHook = '' + cat ${exportYAML stackConfig} > stack-to-nix.yaml + + for f in $(find * -name package.yaml); do + ${snapshot.hpack}/bin/hpack --force $f + done + + echo packages: > cabal.project + for spec in ${concatStringsSep " " localPaths}; do + echo " $spec" >> cabal.project + done + ''; + }; +in + +if shell then shellEnv else target diff --git a/pkgs/stack-to-nix/cabal-name/cabal-name.cabal b/pkgs/stack-to-nix/cabal-name/cabal-name.cabal new file mode 100644 index 0000000000..09b8f429b6 --- /dev/null +++ b/pkgs/stack-to-nix/cabal-name/cabal-name.cabal @@ -0,0 +1,9 @@ +name: cabal-name +version: 1 +cabal-version: >= 1.2 + +executable cabal-name + build-depends: base, Cabal + default-language: Haskell2010 + hs-source-dirs: src + main-is: Main.hs diff --git a/pkgs/stack-to-nix/cabal-name/default.nix b/pkgs/stack-to-nix/cabal-name/default.nix new file mode 100644 index 0000000000..77513fbf33 --- /dev/null +++ b/pkgs/stack-to-nix/cabal-name/default.nix @@ -0,0 +1,19 @@ +{ pkgs ? import {} }: with pkgs; + +let + project = { mkDerivation, base, Cabal }: mkDerivation { + pname = "cabal-name"; + version = "1"; + + src = lib.cleanSource ./.; + + isExecutable = true; + isLibrary = false; + + executableHaskellDepends = [ base Cabal ]; + + license = lib.licenses.free; + }; +in + +haskellPackages.callPackage project {} diff --git a/pkgs/stack-to-nix/cabal-name/src/Main.hs b/pkgs/stack-to-nix/cabal-name/src/Main.hs new file mode 100644 index 0000000000..d303064d23 --- /dev/null +++ b/pkgs/stack-to-nix/cabal-name/src/Main.hs @@ -0,0 +1,14 @@ +module Main where + +import Distribution.PackageDescription.Parsec +import Distribution.Types.GenericPackageDescription +import Distribution.Types.PackageDescription +import Distribution.Types.PackageId +import Distribution.Types.PackageName +import Distribution.Verbosity +import System.Environment + +main = do + args <- getArgs + desc <- readGenericPackageDescription normal (head args) + print $ (unPackageName . pkgName . package . packageDescription) desc diff --git a/pkgs/stack-to-nix/default.nix b/pkgs/stack-to-nix/default.nix new file mode 100644 index 0000000000..518df7951a --- /dev/null +++ b/pkgs/stack-to-nix/default.nix @@ -0,0 +1,17 @@ +{ pkgs ? import {} }: + +{ overrides ? (_: _: {}) +, root +, shell ? false +, stackage ? fetchGit { url = "https://github.com/typeable/nixpkgs-stackage"; rev = "6042df5e646d65b826add0a85d16304bee8e1dd5"; } }: + +let + lib = import ./lib.nix pkgs; + + buildProject = import ./build.nix { + pkgs = pkgs // { lib = pkgs.lib // lib; }; + inherit overrides shell stackage; + }; +in + +buildProject (lib.importYAML "${root}/stack.yaml") root diff --git a/pkgs/stack-to-nix/handlers.nix b/pkgs/stack-to-nix/handlers.nix new file mode 100644 index 0000000000..d52dd436b8 --- /dev/null +++ b/pkgs/stack-to-nix/handlers.nix @@ -0,0 +1,36 @@ +pkgs: with pkgs; with lib; + +[ + ({ + handle = spec: self: + let + components = splitString "-" spec; + name = concatStringsSep "-" (init components); + version = last components; + in + { "${name}" = lib.callHackage self name version {}; }; + + test = isString; + }) + ({ + handle = spec: self: + let + src = fetchGit { + url = spec.git; + ref = "*"; + rev = spec.commit; + }; + + subdirToAttr = subdir: + let + name = cabalPackageName "${src}/${subdir}"; + in + nameValuePair name (cabalToNix self name src {} ''--subpath="${subdir}"''); + + subdirs = spec.subdirs or [ "." ]; + in + listToAttrs (map subdirToAttr subdirs); + + test = spec: isAttrs spec && spec ? git; + }) +] diff --git a/pkgs/stack-to-nix/lib.nix b/pkgs/stack-to-nix/lib.nix new file mode 100644 index 0000000000..6ab4909268 --- /dev/null +++ b/pkgs/stack-to-nix/lib.nix @@ -0,0 +1,55 @@ +pkgs: with pkgs; with lib; + +let + inherit (haskell.lib) overrideCabal; + inherit (haskellPackages) hackage2nix haskellSrc2nix hpack; + + cabal-name = import ./cabal-name { inherit pkgs; }; + + cabalName = path: runCommand "cabal-name" {} '' + ${cabal-name}/bin/cabal-name ${path} > $out + ''; + + hpackToCabal = path: runCommand "hpack.cabal" {} '' + cd ${dirOf path}; ${hpack}/bin/hpack - < ${path} > $out + ''; + + listDirectory = path: + map (name: "${path}/${name}") (attrNames (builtins.readDir path)); + + yamlToJSON = path: runCommand "yaml.json" { nativeBuildInputs = [ ruby ]; } '' + ruby -rjson -ryaml -e "puts YAML.load(ARGF).to_json" < ${path} > $out + ''; +in + +{ + cabalPackageName = root: + let + children = listDirectory root; + hpack = findFirst (hasSuffix "/package.yaml") + (throw "no Cabal or Hpack file found: ${root}") children; + cabal = findSingle (hasSuffix ".cabal") (hpackToCabal hpack) + (throw "more than one Cabal file: ${root}") children; + in + import (cabalName cabal); + + cabalToNix = self: name: src: args: options: + let + expr = haskellSrc2nix { + inherit name src; + extraCabal2nixOptions = options; + }; + in + overrideCabal + (self.callPackage expr args) + (lib.const { inherit src; }); + + callHackage = self: name: version: + self.callPackage (hackage2nix name version); + + exportYAML = term: writeText "term.yaml" (builtins.toJSON term); + + importYAML = path: lib.importJSON (yamlToJSON path); + + mergeExtensions = extensions: foldr composeExtensions (_: _: {}) extensions; +}