Fix code coverage (#8694)

* use latest master
* show the content of $(Pipeline.Workspace)
* Copy the newly downloaded coverage reports
* copy coverage artifacts
* now copy on one line
* create .nyc_output folder
* Check the content of Pipeline.Workspace
* ls -la instead of ls -dlU
* copy nyc_output
* whoops copied one level too deep
* Copy .nyc_output without creating folder first
* Re-enable generate_upload_coverage
* Reset some pipeline changes
* Undo PR validation pipeline change
* Re-add reports stage to PR validation
* Re-add coverage step to CI pipelines
* Whitespace
* Whitespace

Author: kimadeline

build/ci/templates/jobs/coverage.yml

@@ -0,0 +1,8 @@
+jobs:
+- job: Coverage
+  pool:
+    vmImage: "ubuntu-16.04"
+  variables:
+    TestsToRun: 'testUnitTests'
+  steps:
+    - template: ../steps/merge_upload_coverage.yml

build/ci/templates/steps/generate_upload_coverage.yml

@@ -15,7 +15,7 @@ steps:
         summaryFileLocation: "$(System.DefaultWorkingDirectory)/coverage/cobertura-coverage.xml"
         reportDirectory: "$(System.DefaultWorkingDirectory)/coverage"
 
-    - bash: cat ./coverage/lcov.info | ./node_modules/.bin/codecov --pipe
+    - bash: cat ./coverage/cobertura-coverage.xml | ./node_modules/.bin/codecov --pipe
       displayName: 'Upload coverage to codecov'
       continueOnError: true
       condition: contains(variables['TestsToRun'], 'testUnitTests')

build/ci/templates/steps/merge_upload_coverage.yml

@@ -0,0 +1,29 @@
+steps:
+    - template: initialization.yml
+      parameters:
+        workingDirectory: $(Build.SourcesDirectory)
+        compile: 'false'
+
+    # Download previously generated coverage artifacts
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        patterns: '**/.nyc_output/**'
+      displayName: "Download .nyc_output coverage artifacts"
+      condition: always()     
+
+    # Now that we have downloaded artifacts from `coverage-output-`, copy them
+    #   into the root directory (they'll go under `.nyc_output/...`)
+    # These are the coverage output files that can be merged and then we can generate a report from them.
+    # This step results in downloading all individual `./nyc_output` results in coverage
+    #   from all different test outputs.
+    # Running the process of generating reports, will result in generation of
+    #   reports from all coverage data, i.e. we're basically combining all to generate a single merged report.
+    - bash: |
+        cp -r $(Pipeline.Workspace)/coverage-output-*/.nyc_output/ ./
+        cd .nyc_output/
+        ls -dlU .*/ */
+        ls      
+      displayName: "Copy ./.nyc_output"
+      condition: always()
+      
+    - template: generate_upload_coverage.yml

build/ci/vscode-python-ci.yaml

@@ -296,3 +296,10 @@ stages:
     steps:
       - template: templates/test_phases.yml
 
+- stage: Reports
+  dependsOn:
+  - Smoke
+  - Tests
+  condition: always()
+  jobs:
+  - template: templates/jobs/coverage.yml

build/ci/vscode-python-nightly-ci.yaml

@@ -464,3 +464,10 @@ stages:
     steps:
       - template: templates/test_phases.yml
 
+- stage: Reports
+  dependsOn:
+  - Smoke
+  - Tests
+  condition: always()
+  jobs:
+  - template: templates/jobs/coverage.yml

build/ci/vscode-python-pr-validation.yaml

@@ -123,3 +123,11 @@ stages:
 
     steps:
       - template: templates/test_phases.yml
+
+- stage: Reports
+  dependsOn:
+  - Smoke
+  - Tests
+  condition: always()
+  jobs:
+  - template: templates/jobs/coverage.yml

package.json

@@ -2678,7 +2678,7 @@
         "test:unittests:cover": "nyc --silent --no-clean --nycrc-path build/.nycrc mocha --opts ./build/.mocha.unittests.ts.opts",
         "test:functional": "mocha --require source-map-support/register --opts ./build/.mocha.functional.opts",
         "test:functional:cover": "npm run test:functional",
-        "test:cover:report": "nyc --nycrc-path build/.nycrc  report --reporter=lcov --reporter=text --reporter=html --reporter=text-summary --reporter=cobertura",
+        "test:cover:report": "nyc --nycrc-path build/.nycrc  report --reporter=text --reporter=html --reporter=text-summary --reporter=cobertura",
         "testDebugger": "node ./out/test/testBootstrap.js ./out/test/debuggerTest.js",
         "testSingleWorkspace": "node ./out/test/testBootstrap.js ./out/test/standardTest.js",
         "testMultiWorkspace": "node ./out/test/testBootstrap.js ./out/test/multiRootTest.js",