Update HPC for better compatibility with large projects (#1484)

peterbecich · purefn · hamishmack · web-flow · commit 965c7acf2c6f · 2022-09-14T21:24:21.000+12:00
* patch hpc to allow files to be used for the list of src dirs, hpc dirs, and includes

* delete trailing whitespace

* use Response Files for HPC arguments; delete unnecessary patch

* Add ghc patch based on what was merged upstream

* Fix coverage tests

* Update responseFile usage

* Enable coverage-no-libs test

Co-authored-by: Richard Wallace &lt;rwallace@thewallacepack.net&gt;
Co-authored-by: Hamish Mackenzie &lt;Hamish.K.Mackenzie@gmail.com&gt;
diff --git a/lib/cover-project.nix b/lib/cover-project.nix
@@ -8,8 +8,6 @@ project:
 coverageReports:
 
 let
-  toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")";
-
   # Create a list element for a project coverage index page.
   coverageListElement = coverageReport:
       ''
@@ -51,12 +49,16 @@ let
 
   libs = lib.unique (lib.concatMap (r: r.mixLibraries) coverageReports);
 
+  writeArr = name: arr: pkgs.writeText name (lib.concatStringsSep "\n" arr);
+
   mixDirs =
     map
       (l: "${l}/share/hpc/vanilla/mix/${l.identifier.name}-${l.identifier.version}")
       libs;
+  mixDirsFile = writeArr "mixdirs" mixDirs;
 
   srcDirs = map (l: l.srcSubDirPath) libs;
+  srcDirsFile = writeArr "srcdirs" srcDirs;
 
   allCoverageReport = haskellLib.coverageReport {
     name = "all";
@@ -77,45 +79,6 @@ in pkgs.runCommand "project-coverage-report"
     LOCALE_ARCHIVE = "${pkgs.buildPackages.glibcLocales}/lib/locale/locale-archive";
   })
   ''
-    function markup() {
-      local -n srcDs=$1
-      local -n mixDs=$2
-      local -n includedModules=$3
-      local destDir=$4
-      local tixFile=$5
-
-      local hpcMarkupCmd=("hpc" "markup" "--destdir=$destDir")
-      for srcDir in "''${srcDs[@]}"; do
-        hpcMarkupCmd+=("--srcdir=$srcDir")
-      done
-
-      for mixDir in "''${mixDs[@]}"; do
-        hpcMarkupCmd+=("--hpcdir=$mixDir")
-      done
-
-      for module in "''${includedModules[@]}"; do
-        hpcMarkupCmd+=("--include=$module")
-      done
-
-      hpcMarkupCmd+=("$tixFile")
-
-      echo "''${hpcMarkupCmd[@]}"
-      eval "''${hpcMarkupCmd[@]}"
-    }
-
-    function findModules() {
-      local searchDir=$2
-      local pattern=$3
-
-      pushd $searchDir
-      mapfile -d $'\0' $1 < <(find ./ -type f \
-        -wholename "$pattern" -not -name "Paths*" \
-        -exec basename {} \; \
-        | sed "s/\.mix$//" \
-        | tr "\n" "\0")
-      popd
-    }
-
     mkdir -p $out/nix-support
     mkdir -p $out/share/hpc/vanilla/tix/all
     mkdir -p $out/share/hpc/vanilla/mix/
@@ -127,12 +90,14 @@ in pkgs.runCommand "project-coverage-report"
       identifier="${coverageReport.name}"
       report=${coverageReport}
       tix="$report/share/hpc/vanilla/tix/$identifier/$identifier.tix"
-      if test -f "$tix"; then
+      if [ -f "$tix" ]; then
         tixFiles+=("$tix")
       fi
 
       # Copy mix, tix, and html information over from each report
-      cp -Rn $report/share/hpc/vanilla/mix/$identifier $out/share/hpc/vanilla/mix/
+      if [ -d "$report/share/hpc/vanilla/mix/$identifier" ]; then
+        cp -Rn $report/share/hpc/vanilla/mix/$identifier $out/share/hpc/vanilla/mix/
+      fi
       cp -R $report/share/hpc/vanilla/tix/* $out/share/hpc/vanilla/tix/
       cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/
     '') coverageReports)}
diff --git a/lib/cover.nix b/lib/cover.nix
@@ -16,8 +16,6 @@
 }:
 
 let
-  toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")";
-
   mixDir = l: "${l}/share/hpc/vanilla/mix/${l.identifier.name}-${l.identifier.version}";
   mixDirs = map mixDir mixLibraries;
 
@@ -39,130 +37,35 @@ in pkgs.runCommand (name + "-coverage-report")
     LOCALE_ARCHIVE = "${pkgs.buildPackages.glibcLocales}/lib/locale/locale-archive";
   })
   ''
-    function markup() {
-      local -n srcDs=$1
-      local -n mixDs=$2
-      local -n includedModules=$3
-      local destDir=$4
-      local tixFile=$5
-
-      local hpcMarkupCmd=("hpc" "markup" "--destdir=$destDir")
-      for srcDir in "''${srcDs[@]}"; do
-        hpcMarkupCmd+=("--srcdir=$srcDir")
-      done
-
-      for mixDir in "''${mixDs[@]}"; do
-        hpcMarkupCmd+=("--hpcdir=$mixDir")
-      done
-
-      for module in "''${includedModules[@]}"; do
-        hpcMarkupCmd+=("--include=$module")
-      done
-
-      hpcMarkupCmd+=("$tixFile")
-
-      echo "''${hpcMarkupCmd[@]}"
-      eval "''${hpcMarkupCmd[@]}"
-    }
-
-    function sumTix() {
-      local -n includedModules=$1
-      local -n tixFs=$2
-      local outFile="$3"
-
-      if (( "''${#tixFs[@]}" > 0 )); then
-        local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")
-
-        for module in "''${includedModules[@]}"; do
-          hpcSumCmd+=("--include=$module")
-        done
-
-        for tixFile in "''${tixFs[@]}"; do
-          hpcSumCmd+=("$tixFile")
-        done
-
-        echo "''${hpcSumCmd[@]}"
-        eval "''${hpcSumCmd[@]}"
-      else
-        # If there are no tix files we output an empty tix file so that we can
-        # markup an empty HTML coverage report. This is preferable to failing to
-        # output a HTML report.
-        echo 'Tix []' > $outFile
-      fi
-    }
-
-    function findModules() {
-      local -n result=$1
-      local -n searchDirs=$2
-      local pattern=$3
-
-      for dir in "''${searchDirs[@]}"; do
-        pushd $dir
-        local temp=()
-        mapfile -d $'\0' temp < <(find ./ -type f \
-          -wholename "$pattern" -not -name "Paths*" \
-          -exec basename {} \; \
-          | sed "s/\.mix$//" \
-          | tr "\n" "\0")
-        result+=("''${temp[@]}")
-        popd
-      done
-    }
-
     mkdir -p $out/nix-support
     mkdir -p $out/share/hpc/vanilla/mix/
     mkdir -p $out/share/hpc/vanilla/tix/${name}
     mkdir -p $out/share/hpc/vanilla/html/${name}
 
-    local srcDirs=${toBashArray srcDirs}
-    local mixDirs=${toBashArray mixDirs}
-
-    # Copy out mix files used for this report
-    for dir in "''${mixDirs[@]}"; do
-      if [ -d "$dir" ]; then
-        cp -R "$dir" $out/share/hpc/vanilla/mix/
-      fi
-    done
-
-    local mixModules=()
-    # Mix modules for all packages in "mixLibraries"
-    findModules mixModules mixDirs "*.mix"
-
-    # We need to make a distinction between library "exposed-modules" and
-    # "other-modules" used in test suites:
-    #  - "exposed-modules" are addressed as "$library-$version-$hash/module"
-    #  - "other-modules" are addressed as "module"
-    #
-    # This complicates the code required to find the mix modules. For a given mix directory:
-    #
-    # mix
-    # └── ntp-client-0.0.1
-    #     └── ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5
-    #         ├── Network.NTP.Client.mix
-    #         ├── Network.NTP.Client.Packet.mix
-    #         └── Network.NTP.Client.Query.mix
-    #
-    # Iff ntp-client uses "other-modules" in a test suite, both:
-    #   - "mix/ntp-client-0.0.1", and
-    #   - "mix/ntp-client-0.0.1/ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5"
-    # need to be provided to hpc as search directories.
-    #
-    # I'd prefer to just exclude "other-modules", but I can't think of an easy
-    # way to do that in bash.
-    #
-    # Here we expand the search dirs and modify the mix dirs accordingly:
-    for dir in "''${mixDirs[@]}"; do
-      local otherModulesSearchDirs=()
-      # Simply consider any directory with a mix file as a search directory.
-      mapfile -d $'\0' otherModulesSearchDirs < <(find $dir -type f \
-        -wholename "*.mix" \
-        -exec dirname {} \; \
-        | uniq \
-        | tr "\n" "\0")
-      mixDirs+=("''${otherModulesSearchDirs[@]}")
-    done
-
-    local tixFiles=()
+    local srcDirArgs=$(mktemp)
+    ${lib.concatStringsSep "\n" (map (srcDir: ''
+        echo --srcdir=${srcDir} >> $srcDirArgs
+      '') srcDirs)
+    }
+    local mixDirArgs=$(mktemp)
+    ${ # Copy out mix files used for this report
+      lib.concatStrings (map (mixDir: ''
+        local dir=${mixDir}
+        echo --hpcdir=$dir >> $mixDirArgs
+        if [ -d $dir ]; then
+          cp -R "$dir" $out/share/hpc/vanilla/mix/
+        fi
+      '') mixDirs)
+    }
+    local includeArgs=$(mktemp)
+    find $out/share/hpc/vanilla/mix/ -type f \
+      -wholename "*.mix" -not -name "Paths*" \
+      -exec basename {} \; \
+      | sed "s/\.mix$//" \
+      | sed "s/^.*$/--include=\0/" \
+      >> $includeArgs
+
+    local tixFiles=$(mktemp -d)/tixFiles
     ${lib.concatStringsSep "\n" (builtins.map (check: ''
       if [ -d "${check}/share/hpc/vanilla/tix" ]; then
         pushd ${check}/share/hpc/vanilla/tix
@@ -175,11 +78,19 @@ in pkgs.runCommand (name + "-coverage-report")
         cp "$tixFile" "$newTixFile"
 
         # Add the tix file to our list
-        tixFiles+=("$newTixFile")
+        echo $newTixFile >> $tixFiles
 
         # Create a coverage report for *just that check* affecting any of the
         # "mixLibraries"
-        markup srcDirs mixDirs mixModules "$out/share/hpc/vanilla/html/${check.name}/" "$newTixFile"
+        local responseFile=$(mktemp)
+        echo markup > $responseFile
+        echo '--destdir'=$out/share/hpc/vanilla/html/${check.name}/ >> $responseFile
+        cat $srcDirArgs $mixDirArgs $includeArgs >> $responseFile
+        echo $newTixFile >> $responseFile
+
+        echo hpc response file:
+        cat $responseFile
+        hpc @$responseFile
 
         popd
       fi
@@ -192,10 +103,33 @@ in pkgs.runCommand (name + "-coverage-report")
     local markupOutDir="$out/share/hpc/vanilla/html/${name}"
 
     # Sum all of our tix files
-    sumTix mixModules tixFiles "$sumTixFile"
+    if [ -e $tixFiles ]; then
+      local responseFile=$(mktemp)
+      echo sum > $responseFile
+      echo '--union' >> $responseFile
+      echo '--output'=$sumTixFile >> $responseFile
+      cat $includeArgs >> $responseFile
+      cat $tixFiles >> $responseFile
+
+      echo hpc response file:
+      cat $responseFile
+      hpc @$responseFile
+    else
+      # If there are no tix files we output an empty tix file so that we can
+      # markup an empty HTML coverage report. This is preferable to failing to
+      # output a HTML report.
+      echo 'Tix []' > $sumTixFile
+    fi
 
     # Markup a HTML report
-    markup srcDirs mixDirs mixModules "$markupOutDir" "$sumTixFile"
+    local responseFile=$(mktemp)
+    echo markup > $responseFile
+    echo '--destdir'=$markupOutDir >> $responseFile
+    cat $srcDirArgs $mixDirArgs $includeArgs >> $responseFile
+    echo $sumTixFile >> $responseFile
+    echo hpc response file:
+    cat $responseFile
+    hpc @$responseFile
 
     # Provide a HTML zipfile and Hydra links
     ( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
diff --git a/test/coverage-no-libs/default.nix b/test/coverage-no-libs/default.nix
@@ -22,8 +22,7 @@ let
 
 in recurseIntoAttrs ({
   # Does not work on ghcjs because it needs zlib.
-  # TODO projectCoverageReport is broken in master for this example.
-  meta.disabled = true || stdenv.hostPlatform.isGhcjs;
+  meta.disabled = stdenv.hostPlatform.isGhcjs;
   run = stdenv.mkDerivation {
     name = "coverage-test";
 
