* Modifying RazorPrecompiler to use IMemoryCache to cache results when

  precompilation is invoked via Design Time Host.
* Change compilation to occur in parallel

Author: pranavkm

src/Microsoft.AspNet.Mvc.Razor/Razor/PreCompileViews/PrecompilationCacheEntry.cs

@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft Open Technologies, Inc. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using Microsoft.CodeAnalysis;
+
+namespace Microsoft.AspNet.Mvc.Razor
+{
+    /// <summary>
+    /// An entry in the cache used by <see cref="RazorPreCompiler"/>.
+    /// </summary>
+    public class PrecompilationCacheEntry
+    {
+        /// <summary>
+        /// Initializes a new instance of <see cref="PrecompilationCacheEntry"/> for a successful parse.
+        /// </summary>
+        /// <param name="fileInfo">The <see cref="RazorFileInfo"/> of the file being cached.</param>
+        /// <param name="syntaxTree">The <see cref="CodeAnalysis.SyntaxTree"/> to cache.</param>
+        public PrecompilationCacheEntry([NotNull] RazorFileInfo fileInfo,
+                                        [NotNull] SyntaxTree syntaxTree)
+        {
+            FileInfo = fileInfo;
+            SyntaxTree = syntaxTree;
+        }
+
+        /// <summary>
+        /// Initializes a new instance of <see cref="PrecompilationCacheEntry"/> for a failed parse.
+        /// </summary>
+        /// <param name="diagnostics">The <see cref="IReadOnlyList{Diagnostic}"/> produced from parsing the Razor
+        /// file. This does not contain <see cref="Diagnostic"/>s produced from compiling the parsed
+        /// <see cref="CodeAnalysis.SyntaxTree"/>.</param>
+        public PrecompilationCacheEntry([NotNull] IReadOnlyList<Diagnostic> diagnostics)
+        {
+            Diagnostics = diagnostics;
+        }
+
+        /// <summary>
+        /// Gets the <see cref="RazorFileInfo"/> associated with this cache entry instance.
+        /// </summary>
+        /// <remarks>
+        /// This property is not <c>null</c> if <see cref="Success"/> is <c>true</c>.
+        /// </remarks>
+        public RazorFileInfo FileInfo { get; }
+
+        /// <summary>
+        /// Gets the <see cref="SyntaxTree"/> produced from parsing the Razor file.
+        /// </summary>
+        /// <remarks>
+        /// This property is not <c>null</c> if <see cref="Success"/> is <c>true</c>.
+        /// </remarks>
+        public SyntaxTree SyntaxTree { get; }
+
+        /// <summary>
+        /// Gets the <see cref="Diagnostic"/>s produced from parsing the generated contents of the file
+        /// specified by <see cref="FileInfo"/>. This does not contain <see cref="Diagnostic"/>s produced from
+        /// compiling the parsed <see cref="CodeAnalysis.SyntaxTree"/>.
+        /// </summary>
+        /// <remarks>
+        /// This property is <c>null</c> if <see cref="Success"/> is <c>true</c>.
+        /// </remarks>
+        public IReadOnlyList<Diagnostic> Diagnostics { get; }
+
+        /// <summary>
+        /// Gets a value that indicates if parsing was successful.
+        /// </summary>
+        public bool Success
+        {
+            get { return SyntaxTree != null; }
+        }
+    }
+}

src/Microsoft.AspNet.Mvc.Razor/Razor/PreCompileViews/RazorFileInfoCollectionGenerator.cs

@@ -12,16 +12,17 @@ public class RazorFileInfoCollectionGenerator
     {
         private string _fileFormat;
 
-        protected IReadOnlyList<RazorFileInfo> FileInfos { get; private set; }
-        protected CompilationSettings CompilationSettings { get; }
-
-        public RazorFileInfoCollectionGenerator([NotNull] IReadOnlyList<RazorFileInfo> fileInfos,
+        public RazorFileInfoCollectionGenerator([NotNull] IEnumerable<RazorFileInfo> fileInfos,
                                                 [NotNull] CompilationSettings compilationSettings)
         {
             FileInfos = fileInfos;
             CompilationSettings = compilationSettings;
         }
 
+        protected IEnumerable<RazorFileInfo> FileInfos { get; }
+
+        protected CompilationSettings CompilationSettings { get; }
+
         public virtual SyntaxTree GenerateCollection()
         {
             var builder = new StringBuilder();

src/Microsoft.AspNet.Mvc.Razor/Razor/PreCompileViews/RazorPreCompiler.cs

@@ -4,8 +4,11 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
 using Microsoft.AspNet.FileSystems;
-using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis;
+using Microsoft.Framework.Cache.Memory;
 using Microsoft.Framework.DependencyInjection;
 using Microsoft.Framework.OptionsModel;
 using Microsoft.Framework.Runtime;
@@ -17,37 +20,42 @@ public class RazorPreCompiler
     {
         private readonly IServiceProvider _serviceProvider;
         private readonly IFileSystem _fileSystem;
-        private readonly IMvcRazorHost _host;
 
         public RazorPreCompiler([NotNull] IServiceProvider designTimeServiceProvider,
+                                [NotNull] IMemoryCache precompilationCache,
                                 [NotNull] CompilationSettings compilationSettings) :
             this(designTimeServiceProvider,
-                 designTimeServiceProvider.GetRequiredService<IMvcRazorHost>(),
                  designTimeServiceProvider.GetRequiredService<IOptions<RazorViewEngineOptions>>(),
+                 precompilationCache,
                  compilationSettings)
         {
         }
 
         public RazorPreCompiler([NotNull] IServiceProvider designTimeServiceProvider,
-                                [NotNull] IMvcRazorHost host,
                                 [NotNull] IOptions<RazorViewEngineOptions> optionsAccessor,
+                                [NotNull] IMemoryCache precompilationCache,
                                 [NotNull] CompilationSettings compilationSettings)
         {
             _serviceProvider = designTimeServiceProvider;
-            _host = host;
             _fileSystem = optionsAccessor.Options.FileSystem;
             CompilationSettings = compilationSettings;
+            PreCompilationCache = precompilationCache;
         }
 
         protected CompilationSettings CompilationSettings { get; }
 
+        protected IMemoryCache PreCompilationCache { get; }
+
         protected virtual string FileExtension { get; } = ".cshtml";
 
+        protected virtual int MaxDegreesOfParallelism { get; } = Environment.ProcessorCount;
+
+
         public virtual void CompileViews([NotNull] IBeforeCompileContext context)
         {
             var descriptors = CreateCompilationDescriptors(context);
 
-            if (descriptors.Count > 0)
+            if (descriptors.Any())
             {
                 var collectionGenerator = new RazorFileInfoCollectionGenerator(
                                                 descriptors,
@@ -58,93 +66,127 @@ public virtual void CompileViews([NotNull] IBeforeCompileContext context)
             }
         }
 
-        protected virtual IReadOnlyList<RazorFileInfo> CreateCompilationDescriptors(
-                                                            [NotNull] IBeforeCompileContext context)
+        protected virtual IEnumerable<RazorFileInfo> CreateCompilationDescriptors(
+            [NotNull] IBeforeCompileContext context)
         {
-            var list = new List<RazorFileInfo>();
+            var filesToProcess = new List<RelativeFileInfo>();
+            GetFileInfosRecursive(root: string.Empty, razorFiles: filesToProcess);
 
-            foreach (var info in GetFileInfosRecursive(string.Empty))
-            {
-                var descriptor = ParseView(info, context);
+            var razorFiles = new RazorFileInfo[filesToProcess.Count];
+            var syntaxTrees = new SyntaxTree[filesToProcess.Count];
+            var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = MaxDegreesOfParallelism };
+            var diagnosticsLock = new object();
 
-                if (descriptor != null)
+            Parallel.For(0, filesToProcess.Count, parallelOptions, index =>
+            {
+                var file = filesToProcess[index];
+                var cacheEntry = PreCompilationCache.GetOrSet(file.RelativePath,
+                                                              file,
+                                                              OnCacheMiss);
+                if (cacheEntry != null)
                 {
-                    list.Add(descriptor);
+                    if (cacheEntry.Success)
+                    {
+                        syntaxTrees[index] = cacheEntry.SyntaxTree;
+                        razorFiles[index] = cacheEntry.FileInfo;
+                    }
+                    else
+                    {
+                        lock (diagnosticsLock)
+                        {
+                            foreach (var diagnostic in cacheEntry.Diagnostics)
+                            {
+                                context.Diagnostics.Add(diagnostic);
+                            }
+                        }
+                    }
                 }
-            }
+            });
+
+            context.CSharpCompilation = context.CSharpCompilation
+                                               .AddSyntaxTrees(syntaxTrees.Where(tree => tree != null));
+            return razorFiles.Where(file => file != null);
+        }
 
-            return list;
+        protected IMvcRazorHost GetRazorHost()
+        {
+            return _serviceProvider.GetRequiredService<IMvcRazorHost>();
         }
 
-        private IEnumerable<RelativeFileInfo> GetFileInfosRecursive(string currentPath)
+        private PrecompilationCacheEntry OnCacheMiss(ICacheSetContext cacheSetContext)
         {
-            var path = currentPath;
+            var fileInfo = (RelativeFileInfo)cacheSetContext.State;
+            var entry = GetCacheEntry(fileInfo);
 
-            var fileInfos = _fileSystem.GetDirectoryContents(path);
-            if (!fileInfos.Exists)
+            if (entry != null)
             {
-                yield break;
+                cacheSetContext.AddExpirationTrigger(_fileSystem.Watch(fileInfo.RelativePath));
+                foreach (var viewStartPath in ViewStartUtility.GetViewStartLocations(fileInfo.RelativePath))
+                {
+                    cacheSetContext.AddExpirationTrigger(_fileSystem.Watch(viewStartPath));
+                }
             }
 
+            return entry;
+        }
+
+        private void GetFileInfosRecursive(string root, List<RelativeFileInfo> razorFiles)
+        {
+            var fileInfos = _fileSystem.GetDirectoryContents(root);
+
             foreach (var fileInfo in fileInfos)
             {
                 if (fileInfo.IsDirectory)
                 {
-                    var subPath = Path.Combine(path, fileInfo.Name);
-
-                    foreach (var info in GetFileInfosRecursive(subPath))
-                    {
-                        yield return info;
-                    }
+                    var subPath = Path.Combine(root, fileInfo.Name);
+                    GetFileInfosRecursive(subPath, razorFiles);
                 }
                 else if (Path.GetExtension(fileInfo.Name)
                          .Equals(FileExtension, StringComparison.OrdinalIgnoreCase))
                 {
-                    var relativePath = Path.Combine(currentPath, fileInfo.Name);
+                    var relativePath = Path.Combine(root, fileInfo.Name);
                     var info = new RelativeFileInfo(fileInfo, relativePath);
-                    yield return info;
+                    razorFiles.Add(info);
                 }
             }
         }
 
-        protected virtual RazorFileInfo ParseView([NotNull] RelativeFileInfo fileInfo,
-                                                  [NotNull] IBeforeCompileContext context)
+        protected virtual PrecompilationCacheEntry GetCacheEntry([NotNull] RelativeFileInfo fileInfo)
         {
             using (var stream = fileInfo.FileInfo.CreateReadStream())
             {
-                var results = _host.GenerateCode(fileInfo.RelativePath, stream);
-
-                foreach (var parserError in results.ParserErrors)
-                {
-                    var diagnostic = parserError.ToDiagnostics(fileInfo.FileInfo.PhysicalPath);
-                    context.Diagnostics.Add(diagnostic);
-                }
-
-                var generatedCode = results.GeneratedCode;
+                var host = GetRazorHost();
+                var results = host.GenerateCode(fileInfo.RelativePath, stream);
 
-                if (generatedCode != null)
+                if (results.Success)
                 {
-                    var syntaxTree = SyntaxTreeGenerator.Generate(generatedCode,
+                    var syntaxTree = SyntaxTreeGenerator.Generate(results.GeneratedCode,
                                                                   fileInfo.FileInfo.PhysicalPath,
                                                                   CompilationSettings);
-                    var fullTypeName = results.GetMainClassName(_host, syntaxTree);
+                    var fullTypeName = results.GetMainClassName(host, syntaxTree);
 
                     if (fullTypeName != null)
                     {
-                        context.CSharpCompilation = context.CSharpCompilation.AddSyntaxTrees(syntaxTree);
-
                         var hash = RazorFileHash.GetHash(fileInfo.FileInfo);
-
-                        return new RazorFileInfo()
+                        var razorFileInfo = new RazorFileInfo
                         {
-                            FullTypeName = fullTypeName,
                             RelativePath = fileInfo.RelativePath,
                             LastModified = fileInfo.FileInfo.LastModified,
                             Length = fileInfo.FileInfo.Length,
-                            Hash = hash,
+                            FullTypeName = fullTypeName
                         };
+
+                        return new PrecompilationCacheEntry(razorFileInfo, syntaxTree);
                     }
                 }
+                else
+                {
+                    var diagnostics = results.ParserErrors
+                                             .Select(error => error.ToDiagnostics(fileInfo.FileInfo.PhysicalPath))
+                                             .ToList();
+
+                    return new PrecompilationCacheEntry(diagnostics);
+                }
             }
 
             return null;

src/Microsoft.AspNet.Mvc.Razor/project.json

@@ -9,7 +9,8 @@
         "Microsoft.AspNet.Mvc.Core": "6.0.0-*",
         "Microsoft.AspNet.Mvc.Razor.Host": "6.0.0-*",
         "Microsoft.CodeAnalysis.CSharp": "1.0.0-rc1-*",
-        "Microsoft.Framework.Runtime.Roslyn.Common":  "1.0.0-*"
+        "Microsoft.Framework.Cache.Memory": "1.0.0-*",
+        "Microsoft.Framework.Runtime.Roslyn.Common": "1.0.0-*"
     },
     "frameworks": {
         "aspnet50": {
@@ -21,6 +22,10 @@
                 "System.Threading.Tasks": ""
             }
         },
-        "aspnetcore50": {}
+        "aspnetcore50": {
+            "dependencies": {
+                "System.Threading.Tasks.Parallel": "4.0.0-beta-*"
+            }
+        }
     }
 }

src/Microsoft.AspNet.Mvc/RazorPreCompileModule.cs

@@ -9,6 +9,7 @@
 using Microsoft.AspNet.Mvc.Razor;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.Framework.Cache.Memory;
 using Microsoft.Framework.DependencyInjection;
 using Microsoft.Framework.DependencyInjection.Fallback;
 using Microsoft.Framework.DependencyInjection.ServiceLookup;
@@ -19,10 +20,15 @@ namespace Microsoft.AspNet.Mvc
     public abstract class RazorPreCompileModule : ICompileModule
     {
         private readonly IServiceProvider _appServices;
+        private readonly IMemoryCache _memoryCache;
 
         public RazorPreCompileModule(IServiceProvider services)
         {
             _appServices = services;
+            // When ListenForMemoryPressure is true, the MemoryCache evicts items at every gen2 collection.
+            // In DTH, gen2 happens frequently enough to make it undesirable for caching precompilation results. We'll
+            // disable listening for memory pressure for the MemoryCache instance used by precompilation.
+            _memoryCache = new MemoryCache(new MemoryCacheOptions { ListenForMemoryPressure = false });
         }
 
         protected virtual string FileExtension { get; } = ".cshtml";
@@ -39,7 +45,7 @@ public virtual void BeforeCompile(IBeforeCompileContext context)
             sc.AddMvc();
 
             var serviceProvider = BuildFallbackServiceProvider(sc, _appServices);
-            var viewCompiler = new RazorPreCompiler(serviceProvider, compilationSettings);
+            var viewCompiler = new RazorPreCompiler(serviceProvider, _memoryCache, compilationSettings);
             viewCompiler.CompileViews(context);
         }