diff --git a/builder/comp-builder.nix b/builder/comp-builder.nix index df0510b0f9..f218a72cbd 100644 --- a/builder/comp-builder.nix +++ b/builder/comp-builder.nix @@ -229,7 +229,7 @@ stdenv.mkDerivation ({ buildInputs = component.libs ++ component.frameworks - ++ component.pkgconfig; + ++ builtins.concatLists component.pkgconfig; nativeBuildInputs = [ghc buildPackages.removeReferencesTo] diff --git a/builder/default.nix b/builder/default.nix index 11b164b89e..2483fabe79 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -50,26 +50,13 @@ let setup = if package.buildType == "Simple" then defaultSetup - else stdenv.mkDerivation { - name = "${name}-setup"; - nativeBuildInputs = [buildGHC]; - inherit src; - phases = ["unpackPhase" "buildPhase" "installPhase"]; - buildPhase = '' - for f in Setup.hs Setup.lhs; do - if [ -f $f ]; then - echo Compiling package $f - ghc $f --make -o ./Setup - setup=$(pwd)/Setup - fi - done - [ -f ./Setup ] || (echo Failed to build Setup && exit 1) - ''; - - installPhase = '' - mkdir -p $out/bin - install ./Setup $out/bin/Setup - ''; + else haskellLib.weakCallPackage pkgs ./setup-builder.nix { + ghc = buildGHC; + setup-depends = package.setup-depends; + hsPkgs = hsPkgs.buildPackages; + inherit haskellLib nonReinstallablePkgs withPackage + package name src flags pkgconfig + ; }; comp-builder = haskellLib.weakCallPackage pkgs ./comp-builder.nix { inherit ghc haskellLib nonReinstallablePkgs withPackage hsPkgs; }; diff --git a/builder/setup-builder.nix b/builder/setup-builder.nix new file mode 100644 index 0000000000..410081dde8 --- /dev/null +++ b/builder/setup-builder.nix @@ -0,0 +1,110 @@ +{ stdenv, buildPackages, ghc, lib, pkgconfig, writeText, runCommand +, haskellLib, nonReinstallablePkgs, withPackage, hsPkgs, setup-depends +, package, name, src, flags }: + +let + fullName = "${name}-setup"; + + flagsAndConfig = field: xs: lib.optionalString (xs != []) '' + echo ${lib.concatStringsSep " " (map (x: "--${field}=${x}") xs)} >> $out/configure-flags + echo "${field}: ${lib.concatStringsSep " " xs}" >> $out/cabal.config + ''; + + flatDepends = + let + makePairs = map (p: rec { key="${val}"; val=(p.components.library or p); }); + closure = builtins.genericClosure { + startSet = makePairs setup-depends; + operator = {val,...}: makePairs val.config.depends; + }; + in map ({val,...}: val) closure; + + includeGhcPackage = lib.any (p: p.identifier.name == "ghc") setup-depends; + + configFiles = runCommand "${fullName}-config" { nativeBuildInputs = [ghc]; } ('' + mkdir -p $out + + # Calls ghc-pkg for the target platform + target-pkg() { + ${ghc.targetPrefix}ghc-pkg "$@" + } + + target-pkg init $out/package.conf.d + + # Copy over the nonReinstallablePkgs from the global package db. + # Note: we need to use --global-package-db with ghc-pkg to prevent it + # from looking into the implicit global package db when registering the package. + ${lib.concatMapStringsSep "\n" (p: '' + target-pkg describe ${p} | target-pkg --force --global-package-db $out/package.conf.d register - || true + '') nonReinstallablePkgs} + + ${lib.concatMapStringsSep "\n" (p: '' + target-pkg --package-db ${p}/package.conf.d dump | target-pkg --force --package-db $out/package.conf.d register - + '') flatDepends} + + # Note: we pass `clear` first to ensure that we never consult the implicit global package db. + ${flagsAndConfig "package-db" ["clear" "$out/package.conf.d"]} + + echo ${lib.concatStringsSep " " (lib.mapAttrsToList (fname: val: "--flags=${lib.optionalString (!val) "-" + fname}") flags)} >> $out/configure-flags + + '' + # This code originates in the `generic-builder.nix` from nixpkgs. However GHC has been fixed + # to drop unused libraries referneced from libraries; and this patch is usually included in the + # nixpkgs's GHC builds. This doesn't sadly make this stupid hack unnecessary. It resurfes in + # the form of Cabal trying to be smart. Cabal when linking a library figures out that you likely + # need those `rpath` entries, and passes `-optl-Wl,-rpath,...` for each dynamic library path to + # GHC, thus subverting the linker and forcing it to insert all those RPATHs weather or not they + # are needed. We therfore reuse the linker hack here to move all al dynamic lirbaries into a + # common folder (as links) and thus prevent Cabal from going nuts. + # + # TODO: Fix Cabal. + # TODO: this is only needed if we do dynamic libraries. + + lib.optionalString stdenv.isDarwin '' + # Work around a limit in the macOS Sierra linker on the number of paths + # referenced by any one dynamic library: + # + # Create a local directory with symlinks of the *.dylib (macOS shared + # libraries) from all the dependencies. + local dynamicLinksDir="$out/lib/links" + mkdir -p $dynamicLinksDir + for d in $(grep dynamic-library-dirs "$out/package.conf.d/"*|awk '{print $2}'|sort -u); do + ln -s "$d/"*.dylib $dynamicLinksDir + done + # Edit the local package DB to reference the links directory. + for f in "$out/package.conf.d/"*.conf; do + sed -i "s,dynamic-library-dirs: .*,dynamic-library-dirs: $dynamicLinksDir," $f + done + '' + '' + target-pkg --package-db $out/package.conf.d recache + '' + '' + target-pkg --package-db $out/package.conf.d check + ''); + +in stdenv.lib.fix (drv: + +stdenv.mkDerivation { + name = "${fullName}"; + inherit src; + nativeBuildInputs = [ghc]; + + CABAL_CONFIG = configFiles + /cabal.config; + phases = ["unpackPhase" "buildPhase" "installPhase"]; + buildPhase = '' + for f in Setup.hs Setup.lhs; do + if [ -f $f ]; then + echo Compiling package $f + ghc $f '' + (if includeGhcPackage then "-package ghc " else "") + + ''-package-db ${configFiles}/package.conf.d --make -o ./Setup + setup=$(pwd)/Setup + fi + done + [ -f ./Setup ] || (echo Failed to build Setup && exit 1) + ''; + + installPhase = '' + mkdir -p $out/bin + install ./Setup $out/bin/Setup + ''; +}) + + diff --git a/default.nix b/default.nix index a002794923..d1d709a959 100644 --- a/default.nix +++ b/default.nix @@ -6,6 +6,8 @@ # It's also possible to override these sources with NIX_PATH. , hackageSourceJSON ? ./hackage-src.json , stackageSourceJSON ? ./stackage-src.json +, hackageIndexState ? null +, recentNixpkgs ? import {} }: let @@ -38,12 +40,24 @@ let # overridden with NIX_PATH. fetchExternal = import ./lib/fetch-external.nix; + hackageIndex = import ./lib/hackageIndex.nix { + inherit (pkgs) runCommand; + inherit (recentNixpkgs) cabal-install; + indexState = hackageIndexState; + }; + # All packages from Hackage as Nix expressions - hackage = import (fetchExternal { - name = "hackage-exprs-source"; - specJSON = hackageSourceJSON; - override = "hackage"; - }); + hackage = if hackageIndexState == null + then import (fetchExternal { + name = "hackage-exprs-source"; + specJSON = hackageSourceJSON; + override = "hackage"; + }) + else import (import ./lib/callHackageToNix.nix { + inherit (pkgs) runCommand; + inherit (import ./. {}) nix-tools; + inherit hackageIndex; + }); # The set of all Stackage snapshots stackage = import (fetchExternal { @@ -118,6 +132,21 @@ let update-stackage = self.callPackage ./scripts/update-stackage.nix {}; update-pins = self.callPackage ./scripts/update-pins.nix {}; }; + + # Make this handy overridable fetch function available. + inherit fetchExternal; + + # Takes a haskell src directory runs cabal new-configure and plan-to-nix. + # Resulting nix files are added to nix-plan subdirectory. + cabalProjectToNix = import ./lib/cabalProjectToNix.nix { + inherit pkgs hackageIndex; + inherit (pkgs) runCommand; + # TODO avoid hardcoding the GHC version + ghc = recentNixpkgs.haskell.compiler.ghc864; + inherit (recentNixpkgs) cabal-install; + inherit (recentNixpkgs.haskellPackages) hpack; + inherit (import ./. {}) nix-tools; + }; }); in diff --git a/lib/cabalProjectToNix.nix b/lib/cabalProjectToNix.nix new file mode 100644 index 0000000000..35a8922789 --- /dev/null +++ b/lib/cabalProjectToNix.nix @@ -0,0 +1,30 @@ +{ pkgs, runCommand, nix-tools, cabal-install +, hackageIndex, ghc, hpack +} : +{src} : +let + cabalFiles = + pkgs.lib.cleanSourceWith { + inherit src; + filter = path: type: + type == "directory" || + pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; + }; + plan = runCommand "plan" { + buildInputs = [ ghc hpack ]; + } '' + tmp=$(mktemp -d) + cd $tmp + cp -r ${cabalFiles}/* . + chmod +w -R . + find . -name package.yaml -exec hpack "{}" \; + HOME=${hackageIndex} ${cabal-install}/bin/cabal new-configure + HOME=$out ${nix-tools}/bin/plan-to-nix --plan-json dist-newstyle/cache/plan.json -o nix-plan + cp -r nix-plan $out + ''; +in + runCommand "plan-and-src" {} '' + mkdir $out + cp -r ${src}/* $out + ln -sf ${plan} $out/nix-plan + '' diff --git a/lib/callHackageToNix.nix b/lib/callHackageToNix.nix new file mode 100644 index 0000000000..4011ff0161 --- /dev/null +++ b/lib/callHackageToNix.nix @@ -0,0 +1,5 @@ +{ runCommand, nix-tools +, hackageIndex +} : runCommand "hackage-nix" {} '' + HOME=${hackageIndex} ${nix-tools}/bin/hackage-to-nix $out + '' diff --git a/lib/hackageIndex.nix b/lib/hackageIndex.nix new file mode 100644 index 0000000000..d2bc0718a4 --- /dev/null +++ b/lib/hackageIndex.nix @@ -0,0 +1,16 @@ +{ runCommand, cabal-install +, indexState ? "2019-04-24T21:34:04Z" +} : +let + # To avoid downloading more data than necessary this will provide a base. + cachedState = runCommand "hackage-${builtins.substring 0 4 indexState}" {} '' + mkdir -p $out + HOME=$out ${cabal-install}/bin/cabal new-update 'hackage.haskell.org,${builtins.substring 0 4 indexState}-01-01T00:00:00Z' + ''; +in runCommand "hackage-${builtins.replaceStrings [":"] [""] indexState}" {} '' + mkdir -p $out + cp -r ${cachedState}/.cabal $out + chmod +w -R $out/.cabal + sed -i.back -e "s|${cachedState}|$out|g" $out/.cabal/config + HOME=$out ${cabal-install}/bin/cabal new-update 'hackage.haskell.org,${indexState}' + '' diff --git a/lib/pkgconf-nixpkgs-map.nix b/lib/pkgconf-nixpkgs-map.nix index 69114971fb..3da82977f2 100644 --- a/lib/pkgconf-nixpkgs-map.nix +++ b/lib/pkgconf-nixpkgs-map.nix @@ -1,11 +1,166 @@ # pkgconfig entries to nixpkgs map -pkgs: { - cairo-pdf = pkgs.cairo; - cairo-ps = pkgs.cairo; - cairo-svg = pkgs.cairo; - xft = pkgs.xorg.libXft; - xau = pkgs.xorg.libXau; - libR = pkgs.R; - fftw3f = pkgs.fftwFloat; - fftw3 = pkgs.fftw; - } +pkgs: + pkgs.lib.mapAttrs (name: value: [ value ]) pkgs // + { + # Based on https://github.com/NixOS/cabal2nix/blob/11c68fdc79461fb74fa1dfe2217c3709168ad752/src/Distribution/Nixpkgs/Haskell/FromCabal/Name.hs#L23 + "adns" = [ pkgs."adns" ]; + "alsa" = [ pkgs."alsaLib" ]; + "alut" = [ pkgs."freealut" ]; + "appindicator-0.1" = [ pkgs."libappindicator-gtk2" ]; + "appindicator3-0.1" = [ pkgs."libappindicator-gtk3" ]; + "asound" = [ pkgs."alsaLib" ]; + "atk" = [ pkgs."atk" ]; + "b2" = [ pkgs."libb2" ]; + "bz2" = [ pkgs."bzip2" ]; + "c++" = []; # What is that? + "cairo-gobject" = [ pkgs."cairo" ]; + "cairo-pdf" = [ pkgs."cairo" ]; + "cairo-ps" = [ pkgs."cairo" ]; + "cairo-svg" = [ pkgs."cairo" ]; + "crypt" = []; # provided by glibc + "crypto" = [ pkgs."openssl" ]; + "curses" = [ pkgs."ncurses" ]; + "dbusmenu-glib-0.4" = [ pkgs."libdbusmenu" ]; + "dbusmenu-gtk3-0.4" = [ pkgs."libdbusmenu-gtk3" ]; # do we also need pkgs."gtk3" + "dl" = []; # provided by glibc + "fftw3" = [ pkgs."fftw" ]; + "fftw3f" = [ pkgs."fftwFloat" ]; + "gconf" = [ pkgs."GConf" ]; + "gconf-2.0" = [ pkgs."GConf" ]; + "gdk-2.0" = [ pkgs."gtk2" ]; + "gdk-3.0" = [ pkgs."gtk3" ]; + "gdk-pixbuf-2.0" = [ pkgs."gdk_pixbuf" ]; + "gdk-x11-2.0" = [ pkgs."gdk_x11" ]; + "gdk-x11-3.0" = [ pkgs."gtk3" ]; + "gio-2.0" = [ pkgs."glib" ]; + "glib-2.0" = [ pkgs."glib" ]; + "GL" = [ pkgs."libGL" ]; + "GLU" = [ pkgs."libGLU" pkgs."libGL" ]; + "glut" = [ pkgs."freeglut" pkgs."libGLU" pkgs."libGL" ]; + "gnome-keyring" = [ pkgs."gnome-keyring" ]; + "gnome-keyring-1" = [ pkgs."libgnome-keyring" ]; + "gnome-vfs-2.0" = [ pkgs."gnome-vfs" ]; + "gnome-vfs-module-2.0" = [ pkgs."gnome-vfs_module" ]; + "gobject-2.0" = [ pkgs."glib" ]; + "gobject-introspection-1.0" = [ pkgs."gobject-introspection" ]; + "gstreamer-audio-0.10" = [ pkgs."gst-plugins-base" ]; + "gstreamer-audio-1.0" = [ pkgs."gst-plugins-base" ]; + "gstreamer-base-0.10" = [ pkgs."gst-plugins-base" ]; + "gstreamer-base-1.0" = [ pkgs."gst-plugins-base" ]; + "gstreamer-controller-0.10" = [ pkgs."gstreamer" ]; + "gstreamer-dataprotocol-0.10" = [ pkgs."gstreamer" ]; + "gstreamer-net-0.10" = [ pkgs."gst-plugins-base" ]; + "gstreamer-plugins-base-0.10" = [ pkgs."gst-plugins-base" ]; + "gstreamer-video-1.0" = [ pkgs."gst-plugins-base" ]; + "gthread-2.0" = [ pkgs."glib" ]; + "gtk+-2.0" = [ pkgs."gtk2" ]; + "gtk+-3.0" = [ pkgs."gtk3" ]; + "gtk-x11-2.0" = [ pkgs."gtk_x11" ]; + "gtksourceview-3.0" = [ pkgs."gtksourceview3" ]; + "hidapi-libusb" = [ pkgs."hidapi" ]; + "icudata" = [ pkgs."icu" ]; + "icui18n" = [ pkgs."icu" ]; + "icuuc" = [ pkgs."icu" ]; + "idn" = [ pkgs."libidn" ]; + "IL" = [ pkgs."libdevil" ]; + "ImageMagick" = [ pkgs."imagemagick" ]; + "Imlib2" = [ pkgs."imlib2" ]; + "iw" = [ pkgs."wirelesstools" ]; + "jack" = [ pkgs."libjack2" ]; + "javascriptcoregtk-3.0" = [ pkgs."webkitgtk24x-gtk3" ]; # These are the old APIs, of which 2.4 is the last provider, so map directly to that. + "javascriptcoregtk-4.0" = [ pkgs."webkitgtk" ]; + "jpeg" = [ pkgs."libjpeg" ]; + "jvm" = [ pkgs."jdk" ]; + "lapack" = [ pkgs."liblapack" ]; + "lber" = [ pkgs."openldap" ]; + "ldap" = [ pkgs."openldap" ]; + "libavutil" = [ pkgs."ffmpeg" ]; + "libgsasl" = [ pkgs."gsasl" ]; + "libpcre" = [ pkgs."pcre" ]; + "libqrencode" = [ pkgs."qrencode" ]; + "libR" = [ pkgs."R" ]; + "libsoup-gnome-2.4" = [ pkgs."libsoup" ]; + "libsystemd" = [ pkgs."systemd" ]; + "libudev" = [ pkgs."systemd" ]; + "libxml-2.0" = [ pkgs."libxml2" ]; + "libzip" = [ pkgs."libzip" ]; + "libzmq" = [ pkgs."zeromq" ]; + "m" = []; # in stdenv + "magic" = [ pkgs."file" ]; + "MagickWand" = [ pkgs."imagemagick" ]; + "mnl" = [ pkgs."libmnl" ]; + "mpi" = [ pkgs."openmpi" ]; + "ncursesw" = [ pkgs."ncurses" ]; + "netsnmp" = [ pkgs."net_snmp" ]; + "notify" = [ pkgs."libnotify" ]; + "odbc" = [ pkgs."unixODBC" ]; + "openblas" = [ pkgs."openblasCompat" ]; + "panelw" = [ pkgs."ncurses" ]; + "pangocairo" = [ pkgs."pango" ]; + "pcap" = [ pkgs."libpcap" ]; + "pfs-1.2" = [ pkgs."pfstools" ]; + "png" = [ pkgs."libpng" ]; + "poppler-glib" = [ pkgs."poppler" ]; + "pq" = [ pkgs."postgresql" ]; + "pthread" = []; + "pulse" = [ pkgs."libpulseaudio" ]; + "pulse-simple" = [ pkgs."libpulseaudio" ]; + "python-3.3" = [ pkgs."python33" ]; + "python-3.4" = [ pkgs."python34" ]; + "Qt5Core" = [ pkgs."qt5" ]; + "Qt5Gui" = [ pkgs."qt5" ]; + "Qt5Qml" = [ pkgs."qt5" ]; + "Qt5Quick" = [ pkgs."qt5" ]; + "Qt5Widgets" = [ pkgs."qt5" ]; + "quadprog" = [ pkgs."QuadProgpp" ]; + "rt" = []; # in glibc + "rtlsdr" = [ pkgs."rtl-sdr" ]; + "ruby1.8" = [ pkgs."ruby" ]; + "sass" = [ pkgs."libsass" ]; + "sctp" = [ pkgs."lksctp-tools" ]; # This is linux-specific, we should create a common attribute if we ever add sctp support for other systems. + "sdl2" = [ pkgs."SDL2" ]; + "sndfile" = [ pkgs."libsndfile" ]; + "sodium" = [ pkgs."libsodium" ]; + "sqlite3" = [ pkgs."sqlite" ]; + "ssh2" = [ pkgs."libssh2" ]; + "ssl" = [ pkgs."openssl" ]; + "statgrab" = [ pkgs."libstatgrab" ]; + "stdc++" = []; # What is that? + "stdc++.dll" = []; # What is that? + "systemd-journal" = [ pkgs."systemd" ]; + "tag_c" = [ pkgs."taglib" ]; + "taglib_c" = [ pkgs."taglib" ]; + "tensorflow" = [ pkgs."libtensorflow" ]; + "udev" = [ pkgs."systemd" ]; + "uuid" = [ pkgs."libossp_uuid" ]; + "vte-2.91" = [ pkgs."vte_291" ]; + "wayland-client" = [ pkgs."wayland" ]; + "wayland-cursor" = [ pkgs."wayland" ]; + "wayland-egl" = [ pkgs."libGL" ]; + "wayland-server" = [ pkgs."wayland" ]; + "webkit2gtk" = [ pkgs."webkitgtk" ]; + "webkit2gtk-4.0" = [ pkgs."webkitgtk" ]; + "webkit2gtk-web-extension-4.0" = [ pkgs."webkitgtk" ]; + "webkitgtk-3.0" = [ pkgs."webkitgtk24x-gtk3" ]; # These are the old APIs, of which 2.4 is the last provider, so map directly to that + "X11" = [ pkgs."libX11" ]; + "x11" = [ pkgs."xlibsWrapper" ]; + "xau" = [ pkgs."libXau" ]; + "Xcursor" = [ pkgs."libXcursor" ]; + "xerces-c" = [ pkgs."xercesc" ]; + "Xext" = [ pkgs."libXext" ]; + "xft" = [ pkgs."libXft" ]; + "Xi" = [ pkgs."libXi" ]; + "Xinerama" = [ pkgs."libXinerama" ]; + "xkbcommon" = [ pkgs."libxkbcommon" ]; + "xml2" = [ pkgs."libxml2" ]; + "Xpm" = [ pkgs."libXpm" ]; + "Xrandr" = [ pkgs."libXrandr" ]; + "Xrender" = [ pkgs."libXrender" ]; + "Xss" = [ pkgs."libXScrnSaver" ]; + "Xtst" = [ pkgs."libXtst" ]; + "Xxf86vm" = [ pkgs."libXxf86vm" ]; + "yaml" = [ pkgs."libyaml" ]; + "yaml-0.1" = [ pkgs."libyaml" ]; + "z" = [ pkgs."zlib" ]; + "zmq" = [ pkgs."zeromq" ]; +} diff --git a/modules/package.nix b/modules/package.nix index 8690e63771..69bd3e1430 100644 --- a/modules/package.nix +++ b/modules/package.nix @@ -77,6 +77,11 @@ with types; buildType = mkOption { type = str; }; + + setup-depends = mkOption { + type = listOfFilteringNulls unspecified; + default = []; + }; }; components = let @@ -96,7 +101,7 @@ with types; default = []; }; pkgconfig = mkOption { - type = listOfFilteringNulls package; + type = listOf (listOf package); default = []; }; build-tools = mkOption { diff --git a/nix-tools/nix-tools-src.json b/nix-tools/nix-tools-src.json index 2d93547fdf..b0986a1564 100644 --- a/nix-tools/nix-tools-src.json +++ b/nix-tools/nix-tools-src.json @@ -1,7 +1,7 @@ { "url": "https://github.com/input-output-hk/nix-tools", - "rev": "38bf6fd0adef4d22fe06def521f5d793c081f6ed", - "date": "2019-03-20T12:40:31+10:00", - "sha256": "0y8xap5cvc9rssjjvlgv6lyi8ixpxnq675r3gkz2ix7hrsgk8989", + "rev": "19860e5400d713c5d3e57e346f8f0126cf677adb", + "date": "2019-04-30T18:29:23+12:00", + "sha256": "0v343mg1lj0bzk16vc6998blkz8gkyl1l6lvpw837dajiidd5g0x", "fetchSubmodules": false }