Merge dev/release/2.1 into master

Documentation/boostrap-new-os.md

@@ -44,15 +44,14 @@ For versions earlier than .NET Core 2.1, following dependencies are also require
 ### Building the bootstrap CLI
 There is a bash script file that automatizes most of the process of building the bootstrap CLI end to end. The bash script is located at `dotnet/source-build/scripts/bootstrap/buildbootstrapcli.sh`. It first creates a folder named by the new target RID, clones the coreclr, corefx and core-setup repos into it and checks out the same commit of each of the repos as the one that was used to build the seed CLI. This first step is skipped if the coreclr, corefx and core-setup folders already exist. This is important so that the sources can be modified to fix possible build issues and to target the new RID.
 
-The script needs to be passed several arguments. The target architecture, the build configuration, the target OS, the new RID and path to the folder with the untared seed CLI. There is also an optional option to specify the version of the Clang compiler to use to compile the native code. There is also an option to pass in a System.Private.CoreLib.dll built elsewhere and an option to crossgen it. This is useful if you are building debug configuration, since the seed CLI that comes from Azure is built for release configuration and the release version of System.Private.CoreLib.dll is not compatible with debug version of libcoreclr.so.
+The script needs to be passed several arguments. The target architecture, the build configuration, the target OS, the new RID and path to the folder with the untared seed CLI. There is also an optional option to specify the version of the Clang compiler to use to compile the native code. There is also an option to pass in a System.Private.CoreLib.dll built elsewhere. This is useful if you are building debug configuration, since the seed CLI that comes from Azure is built for release configuration and the release version of System.Private.CoreLib.dll is not compatible with debug version of libcoreclr.so.
 Here is the summary of the options that you get from running the script with `--help` option:
 ```
-Usage: buildbootstrapcli.sh [BuildType] --rid <Rid> --seedcli <SeedCli> [--os <OS>] [--clang <Major.Minor>] [--corelib <CoreLib>]
+Usage: buildbootstrapcli.sh [BuildType] -rid <Rid> -seedcli <SeedCli> [-os <OS>] [-clang <Major.Minor>] [-corelib <CoreLib>]
 
 Options:
   BuildType               Type of build (-debug, -release), default: -debug
   -clang <Major.Minor>    Override of the version of clang compiler to use
-  -config <Configuration> Build configuration (debug, release), default: debug
   -corelib <CoreLib>      Path to System.Private.CoreLib.dll, default: use the System.Private.CoreLib.dll from the seed CLI
   -os <OS>                Operating system (used for corefx build), default: Linux
   -rid <Rid>              Runtime identifier including the architecture part (e.g. rhel.6-x64)
@@ -61,7 +60,7 @@ Options:
 ```
 So, for example, when we were creating bootstrap CLI for RHEL / CentOS 6, the command was:
 ```bash
-./buildbootstrapcli.sh -rid rhel.6 -release -os Linux -seedcli ~/seed-cli
+./buildbootstrapcli.sh -release -rid rhel.6-x64 -os Linux -seedcli ~/seed-cli
 ```
 After running the script, check the console output. If the last line printed is `**** Bootstrap CLI was successfully built  ****`, then everything went fine and the bootstrap CLI is ready. You can find it in the `<Rid>-<Architecture>/dotnetcli` subfolder. So for the example command above, it would be `rhel.6-x64/dotnetcli`.
 If there were build errors, they need to be looked into and fixed. After that run the `buildbootstrapcli.sh` with the same arguments again. Repeat until everything builds.

dependencies.props

@@ -5,8 +5,8 @@
     for which dotnet/versions commit was last used to update the dependency.
   -->
   <PropertyGroup>
-    <ProdConCurrentRef>b0f08270da85e75c1fc9ac0b8b56937a4708958e</ProdConCurrentRef>
-    <CoreClrCurrentRef>b0f08270da85e75c1fc9ac0b8b56937a4708958e</CoreClrCurrentRef>
+    <ProdConCurrentRef>79731c509a9d31b921f531ee63c48daa8e0c30d5</ProdConCurrentRef>
+    <CoreClrCurrentRef>79731c509a9d31b921f531ee63c48daa8e0c30d5</CoreClrCurrentRef>
     <BuildToolsCurrentRef>54227d85bec7093210eef26b6013d1435c6fc21e</BuildToolsCurrentRef>
   </PropertyGroup>
 

netci.groovy

@@ -13,8 +13,13 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration)
       }
       else {
         shell("git submodule update --init --recursive");
-        shell("./build.sh /p:Configuration=${configuration} ${loggingOptions}")
-        shell("./smoke-test.sh --minimal --projectOutput --configuration ${configuration}")
+        shell("./build.sh /p:Configuration=${configuration} ${loggingOptions}");
+        smokeTestExcludes = "";
+        if (os == "Fedora24" || os == "OSX10.12") {
+          // Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560
+          smokeTestExcludes += " --excludeWebHttpsTests";
+        }
+        shell("./smoke-test.sh --minimal --projectOutput --configuration ${configuration} ${smokeTestExcludes}");
       }
     };
   };

repos/cli.proj

@@ -94,6 +94,10 @@
       Unable to find package dotnet-watch.
     -->
     <EnvironmentVariables Include="CLIBUILD_SKIP_BUNDLEDDOTNETTOOLS=true" />
+
+    <!-- Produce stable versions for RTM only.  When https://github.com/dotnet/source-build/issues/550
+         is fixed we will be able to determine this automatically.  -->
+    <EnvironmentVariables Include="DropSuffix=true" />
   </ItemGroup>
 
   <Target Name="AddProdConFeed" BeforeTargets="Build">
@@ -113,6 +117,28 @@
                       Overwrite="true" />
   </Target>
 
+  <!--
+    Ignore this error in the CLI build, which doesn't seem to happen in ProdCon:
+
+      error NU5104: A stable release of a package should not have a prerelease dependency. Either
+      modify the version spec of dependency "NuGet.Frameworks [4.7.0-rtm.5156, )" or update the
+      version field in the nuspec.
+  -->
+  <Target Name="AddNoWarnNU5104ByDefault" BeforeTargets="Build">
+    <PropertyGroup>
+      <BuildDefaultsFile>$(ProjectDirectory)build\BuildDefaults.props</BuildDefaultsFile>
+      <ReplacementEndMark>&lt;/NoWarn&gt;</ReplacementEndMark>
+      <InsertItemLine>;NU5104$(ReplacementEndMark)</InsertItemLine>
+
+      <BuildDefaultsPropsContents>$([System.IO.File]::ReadAllText('$(BuildDefaultsFile)'))</BuildDefaultsPropsContents>
+    </PropertyGroup>
+
+    <WriteLinesToFile File="$(BuildDefaultsFile)"
+                      Lines="$(BuildDefaultsPropsContents.Replace('$(ReplacementEndMark)', '$(InsertItemLine)'))"
+                      Condition="! $(BuildDefaultsPropsContents.Contains('$(InsertItemLine)'))"
+                      Overwrite="true" />
+  </Target>
+
   <Target Name="CopyTarBall" AfterTargets="CopyPackage">
     <ItemGroup>
       <CliTarBalls Include="$(TarBallOutput)*$(TarBallExtension)" />

repos/msbuild.proj

@@ -6,7 +6,7 @@
     <!-- Override value from commit to match expected build. -->
     <OfficialBuildId>20180426.1</OfficialBuildId>
     <PackagesOutput>$(ProjectDirectory)/artifacts/$(Configuration)/packages</PackagesOutput>
-    <OutputVersionArgs>/p:VersionPrefix=15.6.82 /p:DisableNerdbankVersioning=true /p:PreReleaseVersionLabel="rc" /p:BUILD_BUILDNUMBER=$(OfficialBuildId)</OutputVersionArgs>
+    <OutputVersionArgs>/p:VersionPrefix=15.7.179 /p:DisableNerdbankVersioning=true /p:PreReleaseVersionLabel="" /p:BUILD_BUILDNUMBER=$(OfficialBuildId)</OutputVersionArgs>
     <BuildCommand>$(ProjectDirectory)build/build$(ShellExtension) build -DotNetBuildFromSource -DotNetCoreSdkDir $(DotNetCliToolDir) -bootstraponly -skiptests -pack -configuration $(Configuration) /p:GitHeadSha=$(GitCommitHash) $(OutputVersionArgs)</BuildCommand>
     <SourceOverrideRepoApiImplemented>true</SourceOverrideRepoApiImplemented>
     <RepoApiImplemented>false</RepoApiImplemented>

scripts/bootstrap/buildbootstrapcli.sh

@@ -5,12 +5,11 @@ set -o pipefail
 usage()
 {
     echo "Builds a bootstrap CLI from sources"
-    echo "Usage: $0 [BuildType] --rid <Rid> --seedcli <SeedCli> [--os <OS>] [--clang <Major.Minor>] [--corelib <CoreLib>]"
+    echo "Usage: $0 [BuildType] -rid <Rid> -seedcli <SeedCli> [-os <OS>] [-clang <Major.Minor>] [-corelib <CoreLib>]"
     echo ""
     echo "Options:"
     echo "  BuildType               Type of build (-debug, -release), default: -debug"
     echo "  -clang <Major.Minor>    Override of the version of clang compiler to use"
-    echo "  -config <Configuration> Build configuration (debug, release), default: debug"
     echo "  -corelib <CoreLib>      Path to System.Private.CoreLib.dll, default: use the System.Private.CoreLib.dll from the seed CLI"
     echo "  -os <OS>                Operating system (used for corefx build), default: Linux"
     echo "  -rid <Rid>              Runtime identifier including the architecture part (e.g. rhel.6-x64)"

smoke-test.sh

@@ -4,30 +4,40 @@ set -euo pipefail
 SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
 TARBALL_PREFIX=dotnet-sdk-
 VERSION_PREFIX=2.1
+DEV_CERTS_VERSION_DEFAULT=2.1.0
 
 projectOutput=false
 keepProjects=false
 dotnetDir=""
 configuration="Release"
+excludeNonWebTests=false
 excludeWebTests=false
+excludeWebNoHttpsTests=false
+excludeWebHttpsTests=false
 excludeLocalTests=false
 excludeOnlineTests=false
+devCertsVersion="$DEV_CERTS_VERSION_DEFAULT"
 testingDir="$SCRIPT_ROOT/testing-smoke"
 cliDir="$testingDir/builtCli"
 logFile="$testingDir/smoke-test.log"
 restoredPackagesDir="$testingDir/packages"
+testingHome="$testingDir/home"
 
 function usage() {
     echo ""
     echo "usage:"
-    echo "  --dotnetDir             the directory from which to run dotnet"
-    echo "  --configuration         the configuration being tested (default=Release)"
-    echo "  --projectOutput         echo dotnet's output to console"
-    echo "  --keepProjects          keep projects after tests are complete"
-    echo "  --minimal               run minimal set of tests - local sources only, no web"
-    echo "  --excludeWebTests       don't run tests for web projects"
-    echo "  --excludeLocalTests     exclude tests that use local sources for nuget packages"
-    echo "  --excludeOnlineTests    exclude test that use online sources for nuget packages"
+    echo "  --dotnetDir                    the directory from which to run dotnet"
+    echo "  --configuration                the configuration being tested (default=Release)"
+    echo "  --projectOutput                echo dotnet's output to console"
+    echo "  --keepProjects                 keep projects after tests are complete"
+    echo "  --minimal                      run minimal set of tests - local sources only, no web"
+    echo "  --excludeNonWebTests           don't run tests for non-web projects"
+    echo "  --excludeWebTests              don't run tests for web projects"
+    echo "  --excludeWebNoHttpsTests       don't run web project tests with --no-https"
+    echo "  --excludeWebHttpsTests         don't run web project tests with https using dotnet-dev-certs"
+    echo "  --excludeLocalTests            exclude tests that use local sources for nuget packages"
+    echo "  --excludeOnlineTests           exclude test that use online sources for nuget packages"
+    echo "  --devCertsVersion <version>    use dotnet-dev-certs <version> instead of default $DEV_CERTS_VERSION_DEFAULT"
     echo ""
 }
 
@@ -57,18 +67,30 @@ while :; do
             keepProjects=true
             ;;
         --minimal)
-            excludeWebTests=true
             excludeOnlineTests=true
             ;;
+        --excludenonwebtests)
+            excludeNonWebTests=true
+            ;;
         --excludewebtests)
             excludeWebTests=true
             ;;
+        --excludewebnohttpstests)
+            excludeWebNoHttpsTests=true
+            ;;
+        --excludewebhttpstests)
+            excludeWebHttpsTests=true
+            ;;
         --excludelocaltests)
             excludeLocalTests=true
             ;;
         --excludeonlinetests)
             excludeOnlineTests=true
             ;;
+        --devcertsversion)
+            shift
+            devCertsVersion="$1"
+            ;;
         *)
             usage
             exit 1
@@ -90,6 +112,24 @@ function doCommand() {
     mkdir "${lang}_${proj}"
     cd "${lang}_${proj}"
 
+    newArgs="new $proj -lang $lang"
+
+    while :; do
+        if [ $# -le 0 ]; then
+            break
+        fi
+        case "$1" in
+            --new-arg)
+                shift
+                newArgs="$newArgs $1"
+                ;;
+            *)
+                break
+                ;;
+        esac
+        shift
+    done
+
     while :; do
         if [ $# -le 0 ]; then
             break
@@ -99,9 +139,9 @@ function doCommand() {
 
         if [ "$1" == "new" ]; then
             if [ "$projectOutput" == "true" ]; then
-                "${dotnetCmd}" new $proj -lang $lang | tee -a "$logFile"
+                "${dotnetCmd}" $newArgs | tee -a "$logFile"
             else
-                "${dotnetCmd}" new $proj -lang $lang >> "$logFile" 2>&1
+                "${dotnetCmd}" $newArgs >> "$logFile" 2>&1
             fi
         elif [[ "$1" == "run" && "$proj" =~ ^(web|mvc|webapi|razor)$ ]]; then
             if [ "$projectOutput" == "true" ]; then
@@ -110,12 +150,20 @@ function doCommand() {
                 "${dotnetCmd}" $1 >> "$logFile" 2>&1 &
             fi
             webPid=$!
-            echo "    waiting 20 seconds for web project with pid $webPid"
-            sleep 20
-            echo "    sending SIGINT to $webPid"
-            pkill -SIGINT -P $webPid
+            killCommand="pkill -SIGTERM -P $webPid"
+            echo "    waiting up to 20 seconds for web project with pid $webPid..."
+            echo "    to clean up manually after an interactive cancellation, run: $killCommand"
+            for seconds in $(seq 20); do
+                if [ "$(tail -n 1 "$logFile")" = 'Application started. Press Ctrl+C to shut down.' ]; then
+                    echo "    app ready for shutdown after $seconds seconds"
+                    break
+                fi
+                sleep 1
+            done
+            echo "    stopping $webPid" | tee -a "$logFile"
+            $killCommand
             wait $!
-            echo "    terminated with exit code $?"
+            echo "    terminated with exit code $?" | tee -a "$logFile"
         else
             if [ "$projectOutput" == "true" ]; then
                 "${dotnetCmd}" $1 | tee -a "$logFile"
@@ -141,35 +189,68 @@ function doCommand() {
     echo "finished language $lang, type $proj" | tee -a smoke-test.log
 }
 
+function setupDevCerts() {
+    echo "Setting up dotnet-dev-certs $devCertsVersion to generate dev certificate" | tee -a "$logFile"
+    (
+        set -x
+        "$dotnetDir/dotnet" tool install -g dotnet-dev-certs --version "$devCertsVersion" --add-source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json
+        export DOTNET_ROOT="$dotnetDir"
+        "$testingHome/.dotnet/tools/dotnet-dev-certs" https
+    ) >> "$logFile" 2>&1
+}
+
 function runAllTests() {
     # Run tests for each language and template
-    doCommand C# console new restore run
-    doCommand C# classlib new restore build
-    doCommand C# xunit new restore test
-    doCommand C# mstest new restore test
-
-    doCommand VB console new restore run
-    doCommand VB classlib new restore build
-    doCommand VB xunit new restore test
-    doCommand VB mstest new restore test
-
-    doCommand F# console new restore run
-    doCommand F# classlib new restore build
-    doCommand F# xunit new restore test
-    doCommand F# mstest new restore test
-
+    if [ "$excludeNonWebTests" == "false" ]; then
+        doCommand C# console new restore run
+        doCommand C# classlib new restore build
+        doCommand C# xunit new restore test
+        doCommand C# mstest new restore test
+
+        doCommand VB console new restore run
+        doCommand VB classlib new restore build
+        doCommand VB xunit new restore test
+        doCommand VB mstest new restore test
+
+        doCommand F# console new restore run
+        doCommand F# classlib new restore build
+        doCommand F# xunit new restore test
+        doCommand F# mstest new restore test
+    fi
+
     if [ "$excludeWebTests" == "false" ]; then
-        doCommand C# web new restore run
-        doCommand C# mvc new restore run
-        doCommand C# webapi new restore run
-        doCommand C# razor new restore run
-
-        doCommand F# web new restore run
-        doCommand F# mvc new restore run
-        doCommand F# webapi new restore run
+        if [ "$excludeWebNoHttpsTests" == "false" ]; then
+            runWebTests --new-arg --no-https
+        fi
+
+        if [ "$excludeWebHttpsTests" == "false" ]; then
+            setupDevCerts
+            runWebTests
+        fi
     fi
 }
 
+function runWebTests() {
+    doCommand C# web "$@" new restore run
+    doCommand C# mvc "$@" new restore run
+    doCommand C# webapi "$@" new restore run
+    doCommand C# razor "$@" new restore run
+
+    doCommand F# web "$@" new restore run
+    doCommand F# mvc "$@" new restore run
+    doCommand F# webapi "$@" new restore run
+}
+
+function resetCaches() {
+    rm -rf "$testingHome"
+    mkdir "$testingHome"
+
+    HOME="$testingHome"
+
+    # clean restore path
+    rm -rf "$restoredPackagesDir"
+}
+
 function setupProdConFeed() {
     sed -i.bakProdCon "s|PRODUCT_CONTRUCTION_PACKAGES|$prodConFeedUrl|g" "$testingDir/NuGet.Config"
 }
@@ -207,6 +288,7 @@ prodConFeedUrl="$(cat "$SCRIPT_ROOT/ProdConFeed.txt")"
 
 # Run all tests, local restore sources first, online restore sources second
 if [ "$excludeLocalTests" == "false" ]; then
+    resetCaches
     # Setup NuGet.Config with local restore source
     if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then
         cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config"
@@ -220,10 +302,8 @@ if [ "$excludeLocalTests" == "false" ]; then
     echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!"
 fi
 
-# clean restore path
-rm -rf "$restoredPackagesDir"
-
 if [ "$excludeOnlineTests" == "false" ]; then
+    resetCaches
     # Setup NuGet.Config to use online restore sources
     if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then
         cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config"

tools-local/ILAsmVersion.txt

@@ -1 +1 @@
-2.1.0-rtm-26514-02
+2.1.0-rtm-26515-03