Merge branch 'dev' into dev-fixes.

This brings dev-fixes up-to-date with dev in preparation for
submitting a new PR to Microsoft.

Author: rchyena

automation/UNIX/build-and-test.sh

@@ -13,8 +13,14 @@ fi
 set -ue
 set -o pipefail
 
-./setup-files.sh
-./run-cmake.sh
-./test-clang.sh
+if [ "$RUN_LOCAL" = "no" ]; then
+  ./setup-files.sh
+  ./run-cmake.sh
+  ./test-clang.sh
+fi
+
 ./test-lnt.sh
-./build-package.sh
+
+if [ "$RUN_LOCAL" = "no" ]; then
+  ./build-package.sh
+fi

automation/UNIX/config-vars.sh

@@ -131,18 +131,24 @@ if [ -z "$BUILD_CPU_COUNT" ]; then
   export BUILD_CPU_COUNT=$(($NPROC*3/4))
 fi
 
+if [ -z "$RUN_LOCAL" ]; then
+  export RUN_LOCAL="no"
+fi
+
 # LLVM Nightly Tests are enabled when LNT is a non-empty
 # string.
 if [ -z "$LNT" ]; then
   # Make sure LNT variable is defined so that scripts that require all variables
   # to be defined do not break.
   export LNT=""
   export LNT_RESULTS_DIR=""
-  export LN_SCRIPT=""
+  export LNT_SCRIPT=""
 else
   export LNT_RESULTS_DIR="${BUILD_BINARIESDIRECTORY}/LNT-Results-${BUILDCONFIGURATION}-${BUILDOS}"
-  # We assume that lnt is installed in /lnt-install on test machines.
-  export LNT_SCRIPT=/lnt-install/sandbox/bin/lnt
+  if [ "$RUN_LOCAL" = "no" ]; then
+    # We assume that lnt is installed in /lnt-install on test machines.
+    export LNT_SCRIPT=/lnt-install/sandbox/bin/lnt
+  fi
 fi
 
 if [ "$CHECKEDC_CONFIG_STATUS" == "passed" ]; then
@@ -156,6 +162,7 @@ if [ "$CHECKEDC_CONFIG_STATUS" == "passed" ]; then
   echo " SKIP_CHECKEDC_TESTS: $SKIP_CHECKEDC_TESTS"
   echo " LNT: $LNT"
   echo " LNT_SCRIPT: $LNT_SCRIPT"
+  echo " RUN_LOCAL: $RUN_LOCAL"
   echo
   echo " Directories:"
   echo "  BUILD_SOURCESDIRECTORY: $BUILD_SOURCESDIRECTORY"

automation/UNIX/run-lnt-local.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# This script can be used to run LNT tests locally on a user's Linux machine.
+# For detailed setup instructions see
+# https://github.com/Microsoft/checkedc-llvm-test-suite/blob/master/README.md
+
+usage() {
+  echo "Usage: SRC_DIR=</path/to/llvm/src> BUILD_DIR=</path/to/llvm/build> $0"
+  echo "Optional flags: TEST_TARGET=\"X86_64;ARM\" LNT_BIN=</path/to/lnt>"
+  exit 1
+}
+
+CURDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+if [ -z "$SRC_DIR" ]; then
+  echo "Specify SRC_DIR as the dir containing LLVM sources."
+  usage
+fi
+
+if [ -z "$BUILD_DIR" ]; then
+  echo "Specify BUILD_DIR as the dir containing LLVM build."
+  usage
+fi
+
+if [ -z "$TEST_TARGET" ]; then
+  TEST_TARGET="X86_64;ARM"
+fi
+
+if [ -z "$LNT_BIN" ]; then
+  LNT_BIN=~/mysandbox/bin/lnt
+fi
+
+rm -rf $BUILD_DIR/LNT-Results-Release-Linux $BUILD_DIR/LLVM-Release-Linux.obj
+ln -s $BUILD_DIR/llvm $BUILD_DIR/LLVM-Release-Linux.obj
+
+if [ ! -d $SRC_DIR/llvm-test-suite ]; then
+  echo "Checking out llvm-test-suite at $SRC_DIR/llvm-test-suite."
+  cd $SRC_DIR
+  git clone https://github.com/microsoft/checkedc-llvm-test-suite.git llvm-test-suite
+fi
+
+cd $CURDIR
+export PATH=$BUILD_DIR/llvm/bin:$PATH
+
+# Invoke lnt.
+BUILDCONFIGURATION=Release \
+BUILD_BINARIESDIRECTORY=$BUILD_DIR \
+BUILD_SOURCESDIRECTORY=$SRC_DIR \
+LNT_SCRIPT=$LNT_BIN \
+TEST_TARGET_ARCH=$TEST_TARGET \
+TEST_SUITE=CheckedC_clang \
+LNT=yes \
+RUN_LOCAL=yes \
+./build-and-test.sh

docs/checkedc/CheckedCConvert.md

@@ -0,0 +1,50 @@
+# Checked-C-Convert
+
+
+This is a clang tool that tries to automatically convert `C` code to include `Checked
+C` types.
+
+The tool is available under: `tools/checked-c-convert`
+
+## Redesigning and new features
+
+Corresponding pull request: [Porting Tool Updates](https://github.com/microsoft/checkedc-clang/pull/642)
+
+In this pull request, we tried to faithfully implement the highlevel idea as described in the paper (Section 5.3 of [https://www.microsoft.com/en-us/research/uploads/prod/2019/05/checkedc-post2019.pdf](https://www.microsoft.com/en-us/research/uploads/prod/2019/05/checkedc-post2019.pdf))
+
+If you see the Section 5.3 of the paper, there depending on how a parameter to a function is **used** by the callers and **used** inside the callee, we use `itype`s or casting. To do this, we need to have two sets of constraint variables for each of the function parameters i.e., one for the calleers (i.e., as seen outside the function) and one for the function body. Once we solve the constraints we can assign `itype` or casts. Similarly for return types.
+
+However, our old implementation had inconsistencies in inferring checked types of the function parameters and returns. Specifically, when a function do not have a corresponding declaration e.g., `static` methods, 
+
+We changed this so that irrespective of whether a function has a explicit declaration or not we always maintain two sets of constraint variables for parameters and returns.
+
+### Function Subtyping
+We introduced the support for `_Nt_array_ptrs`, this resulted in function subtyping issues as discussed in detail in: [https://gist.github.com/Machiry/962bf8c24117bc5f56a31598e6782100](https://gist.github.com/Machiry/962bf8c24117bc5f56a31598e6782100)
+We implemented support for this.
+
+### Performance Improvements
+
+We made performance imporvements to the tool. This is mainly because of a single line change (Thanks to @rchyena) in `ConstraintVariables.cpp`:
+from:
+```
+214: auto env = CS.getVariables();
+```
+to
+```
+214: auto &env = CS.getVariables();
+```
+#### Numbers for Icecast :
+**New**
+```
+real    3m51.869s  
+user    3m50.132s  
+sys    0m0.804s
+```
+**Old**
+```
+real    6m9.581s  
+user    6m0.688s  
+sys    0m8.004s
+```
+In general the current version of the tool is sufficiently fast, for `libarchive` (~170K lines) it took ~10min. 
+

docs/checkedc/Testing.md

@@ -1,32 +1,36 @@
 # Testing the Checked C version of LLVM/clang
 
 LLVM/clang have two kinds of tests: developer regression tests and extended
-tests.  Developer regression tests are meant to be run by developers before any
-check-in and are quick to run.  Extended tests are run as part of continuous
+tests. Developer regression tests are meant to be run by developers before any
+check-in and are quick to run. Extended tests are run as part of continuous
 integration testing or for changes that require extensive testing.
 
 ## Developer regression tests
 
-We have created a new set of developer regression tests for the Checked C extension.
-We have added a new target to the build system for running the test suite: check-checkedc.
-The Checked C version of clang/LLVM should pass the existing clang and LLVM
-regression tests and the Checked C-specific regression tests.   There should
-be no unexpected test failures.  A developer should confirm this before committing a change.
+We have created a new set of developer regression tests for the Checked C
+extension. We have added a new target to the build system for running the test
+suite: check-checkedc. The Checked C version of clang/LLVM should pass the
+existing clang and LLVM regression tests and the Checked C-specific regression
+tests.  There should be no unexpected test failures. A developer should confirm
+this before committing a change.
 
-When testing a change, the testing should be sufficient for the type of change.  For changes
-to parsing and typechecking, it is usually sufficient to pass the Checked C and clang tests.
+When testing a change, the testing should be sufficient for the type of change.
+For changes to parsing and typechecking, it is usually sufficient to pass the
+Checked C and clang tests.
 
 ## Running developer regressions tests
 
 ### From Visual Studio
-Load the solution and the open it using the Solution explorer (View->Solution Explorer).  To run tests, you can right click and build the following targets:
+Load the solution and the open it using the Solution explorer (View->Solution
+Explorer). To run tests, you can right click and build the following targets:
 
 - Checked C tests: go to _CheckedC tests->check-checkedc_
 - clang tests: go to _Clang tests->check-clang_
 - All LLVM and clang tests: select the check-all solution (at the top level)
 
 ### From a command shell using msbuild
-Set up the build system and then change to your new object directory.  Use the following commands to run tests:
+Set up the build system and then change to your new object directory. Use the
+following commands to run tests:
 
 - Checked C tests: `msbuild projects\checkedc-wrapper\check-checkedc.vcxproj /p:CL_MPCount=3 /m`
 - Clang tests: `msbuild tools\clang\test\check-clang.vcxproj /p:CL_MPCount=3 /m`
@@ -36,37 +40,81 @@ Set up the build system and then change to your new object directory.  Use the f
 In your build directory,
 
 - Checked C tests: `make -j nnn check-checkedc`
+- Checked C tests (for ARM target): `make -j nnn check-checkedc-arm`
 - clang tests: `make -j nnn check-clang`
 - All tests: `make -j nnn check-all`
 
 where `nnn` is replaced by the number of CPU cores that your computer has.
 
+Note: If you use CMake with ninja, then you can simply replace `make -j nnn` in
+the above commands with `ninja`. For example:
+
+    `ninja check-checkedc`
+
 ### From a command shell using the testing harness
 You can use the testing harness to run individual tests or sets of tests.
 These directions are adapted from http://llvm.org/docs/TestingGuide.html#id11.
-The clang-specific documentation on running tests appears to be out-of-date, so don't try to follow those directions.  
+The clang-specific documentation on running tests appears to be out-of-date, so
+don't try to follow those directions.
+
+- The build creates a llvm-lit.py script that you can use to run regression
+  tests by hand. It is placed in the same directory as your newly built clang
+executable.
+
+  On Windows, the clang executable resides in:
+    `<BUILD_DIR>\llvm\Debug\bin` for Debug builds
+    `<BUILD_DIR>\llvm\Release\bin` for Release builds
+
+  On Linux, the clang executable resides in:
+    `<BUILD_DIR>\llvm\bin`
+
+- You can point the llvm-lit script at a directory to run all tests or an
+  individual test. For example:
+
+    `llvm-lit <BUILD_DIR>\llvm\tools\clang\test`
+
+### Running lit tests for ARM
+By default, the lit tests are built and run for the native X86 target. But you
+can also build and run them for other targets. Currently, we support lit
+testing for ARM. Here is how you enable ARM lit testing:
+
+- Configure the Checked C compiler
+  Specify these two CMake options when configuring the compiler during the
+build step:
+
+    `-DCHECKEDC_ARM_SYSROOT=<sysroot>` Specify the path to the top of an ARM
+sysroot which contains the ARM-specific headers and runtime libs.
+
+    `-DCHECKEDC_ARM_RUNUNDER=<device>` Specify the device to run the ARM lit
+tests on. For example, specify this as `qemu-arm` in order to run the tests on
+the ARM QEMU.
+
+    NOTE: If you want to run the tests on the QEMU, make sure that the QEMU is
+installed on the host machine.
 
-- The build creates a llvm-lit.py script that you can use to run regression tests by hand.
-  It is placed in the same directory as your newly built clang executable.   Those are placed in {your build directory}\debug\bin. 
+- Run lit for ARM
+  Once you have built the Checked C compiler, this is how you would invoke lit
+for ARM testing:
 
-- You can point the llvm-lit script at a directory to run all tests or an individual test.  For example:
+    `llvm-lit <SRC_DIR>\llvm\projects\checkedc-wrapper\checkedc\tests -DCHECKEDC_TARGET=ARM`
 
-		llvm-lit d:\autobahn1\llvm\tools\clang\test
+NOTE: LIT testing for ARM is only supported for tests under the directory
+`<SRC_DIR>\llvm\projects\checkedc-wrapper\checkedc\tests`.
 
 ### Unexpected test failures and line endings
 
 If you see unexpected test failures when working on Windows using unchanged
 versions of the Checked C LLVM/clang repos, you should first check that you
 have set your line end handling for Git properly, particularly if you see a
-lot of failures.  LLVM and clang have some tests that depend on line endings.
+lot of failures. LLVM and clang have some tests that depend on line endings.
 Those tests assume that all lines end with line feeds (the Unix line ending
 convention).
 
 The LLVM/clang project uses Subversion for source code control, which does not
-alter line endings.  We are using Git for source code control, which may alter
-line endings on Windows, depending on how you have configured Git.  Git may
+alter line endings. We are using Git for source code control, which may alter
+line endings on Windows, depending on how you have configured Git. Git may
 alter text line endings to be carriage return/line feed (the Windows line
-ending convention).  It is important to ensure that Git does not do this for
+ending convention). It is important to ensure that Git does not do this for
 your LLVM/clang repositories.
 
 The configuration setting `core.autocrlf` should to be set to `false`. If you
@@ -75,11 +123,12 @@ it will be set to `false`.
 
 ## Extended testing
 
-The extended testing for LLVM/clang is somewhat complicated to set up.  It
-is oriented toward automated testing using servers and has the set-up complexity
-that comes with that. For the Checked C implementation, there are two kinds
-of extended testing that we do: testing that benchmarks that have been converted
-to Checked C work and testing that the Checked C implementation has not broken existing functionality.
+The extended testing for LLVM/clang is somewhat complicated to set up. It is
+oriented toward automated testing using servers and has the set-up complexity
+that comes with that. For the Checked C implementation, there are two kinds of
+extended testing that we do: testing that benchmarks that have been converted
+to Checked C work and testing that the Checked C implementation has not broken
+existing functionality.
 
 To run benchmarks, clone the
 [Checked C LLVM test suite repo](https://github.com/microsoft/checkedc-llvm-test-suite)
@@ -91,16 +140,17 @@ check out the `original` branch of the
 and then follow the directions in the
 [README.md file](https://github.com/Microsoft/checkedc-llvm-test-suite/blob/master/README.md).
 Note that the  `original` branch contains tests as well as benchmarks.
-It is much larger than the `master` branch.  It will use more than 2 GBytes of disk space
+It is much larger than the `master` branch. It will use more than 2 GBytes of disk space
 and the testing will take much longer to run.
 
 
 ## Gotchas
 
-Windows x86 will cause issues if executables have "install", "setup", or "update" in
-their filename. It will assume the executable is an installer, and require that the user has
-elevated privileges to run the program. Usually inside visual studio this manifests itself
-as "WinError 740". There is more documentation on the heuristics used here:
+Windows x86 will cause issues if executables have "install", "setup", or
+"update" in their filename. It will assume the executable is an installer, and
+require that the user has elevated privileges to run the program. Usually
+inside visual studio this manifests itself as "WinError 740". There is more
+documentation on the heuristics used here:
 https://msdn.microsoft.com/en-us/enus/library/aa905330.aspx
 
 We originally diagnosed this issue thanks to this stackoverflow post:

include/clang/AST/ASTContext.h

@@ -279,6 +279,22 @@ class ASTContext : public RefCountedBase<ASTContext> {
   llvm::DenseMap<const MaterializeTemporaryExpr *, APValue *>
     MaterializedTemporaryValues;
 
+  /// Mapping from (generic record decl, type arguments) pairs to instantiated record decls.
+  /// e.g. (List, int) -> List<int>
+  /// This keeps tracks of all type applications both so we can preserve the uniqueness invariant
+  /// for decls and types, and also so we can typecheck complex recursive applications.
+  llvm::DenseMap<std::pair<const RecordDecl *, ArrayRef<const Type *> >, RecordDecl *>
+    CachedTypeApps;
+
+  /// Mapping from RecordDecls to list of delayed type applications.
+  /// The key is a declaration or definition of the generic RecordDecl, and the
+  /// corresponding values all have the given RecordDecl as base.
+  /// e.g. List<T> -> [List<int>, List<List<char>>, List<char>, ...]
+  ///      Foo<T>  -> [Foo<Foo<int>>, Foo<char>, ...]
+  /// A delayed type application is represented as a RecordDecl for which RecordDecl::isInstantiated()
+  /// returns 'true'. The parameters in a type application can be retrieved via RecordDecl::typeParams().
+  llvm::DenseMap<const RecordDecl *, llvm::SmallVector<RecordDecl *, 4> > DelayedTypeApps;
+
   /// Representation of a "canonical" template template parameter that
   /// is used in canonical template names.
   class CanonicalTemplateTemplateParm : public llvm::FoldingSetNode {
@@ -3074,6 +3090,34 @@ class ASTContext : public RefCountedBase<ASTContext> {
   };
 
   llvm::StringMap<SectionInfo> SectionInfos;
+
+public:
+  //===--------------------------------------------------------------------===//
+  //                     Checked C: type applications
+  //===--------------------------------------------------------------------===//
+
+  /// Get the result of the type application 'Base<TypeArgs>', if it's been already cached.
+  /// If it's not cached, return 'nullptr'.
+  RecordDecl *getCachedTypeApp(const RecordDecl *Base, ArrayRef<const Type *> TypeArgs);
+
+  /// Return all type applications that have the given generic decl as base.
+  /// This is currently slow since it iterates over all cached type applications.
+  /// TODO: improve its efficiency. See issues/644.
+  std::vector<const RecordDecl *> getTypeAppsWithBase(const RecordDecl *Base);
+
+  /// Add the instantiated record type 'Inst' as the result of the type application 'Base<TypeArgs>'.
+  /// Cached applications shouldn't be overwritten, so this should be called at most once per key.
+  void addCachedTypeApp(const RecordDecl *Base, ArrayRef<const Type *> TypeArgs, RecordDecl *Inst);
+
+  /// Get the list of type applications that have 'Base' as their base RecordDecl.
+  ArrayRef<RecordDecl *> getDelayedTypeApps(RecordDecl *Base);
+
+  /// Add TypeApp to the cache using TypeApp's base field as the key.
+  void addDelayedTypeApp(RecordDecl *TypeApp);
+
+  /// Remove all type applications that have 'Base' as their base RecordDecl.
+  /// Return 'true' if the removed key was in the cache, and 'false' otherwise.
+  bool removeDelayedTypeApps(RecordDecl *Base);
 };
 
 /// Utility function for constructing a nullary selector.