Improve source indexer deduplication to prefer real implementation over *.notsupported.cs files

src/SourceBrowser/src/BinLogParser/BinLogReader.cs

@@ -1,9 +1,12 @@
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.IO;
-using Microsoft.Build.Framework;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Logging.StructuredLogger;
 using Microsoft.CodeAnalysis;
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 
 namespace Microsoft.SourceBrowser.BinLogParser
 {
@@ -33,26 +36,63 @@ public static IEnumerable<CompilerInvocation> ExtractInvocations(string binLogFi
 
             var lazyResult = m_binlogInvocationMap.GetOrAdd(binLogFilePath, new Lazy<List<CompilerInvocation>>(() =>
             {
+                // for old format logs, use the legacy reader - this is less desireable because it loads everything into memory
                 if (binLogFilePath.EndsWith(".buildlog", StringComparison.OrdinalIgnoreCase))
                 {
                     return ExtractInvocationsFromBuild(binLogFilePath);
                 }
 
+                // for new format logs, replay the log to avoid loading everything into memory
                 var invocations = new List<CompilerInvocation>();
                 var reader = new Microsoft.Build.Logging.StructuredLogger.BinLogReader();
                 var taskIdToInvocationMap = new Dictionary<(int, int), CompilerInvocation>();
+                var projectEvaluationToPropertiesMap = new Dictionary<int, Dictionary<string, string>>();
+                var projectInstanceToEvaluationMap = new Dictionary<int, int>();
 
                 void TryGetInvocationFromEvent(object sender, BuildEventArgs args)
                 {
-                    var invocation = TryGetInvocationFromRecord(args, taskIdToInvocationMap);
+                    Dictionary<string, string> projectProperties = null;
+                    if (projectInstanceToEvaluationMap.TryGetValue(args.BuildEventContext?.ProjectInstanceId ?? -1, out var evaluationId) &&
+                        projectEvaluationToPropertiesMap.TryGetValue(evaluationId, out var properties))
+                    {
+                        projectProperties = properties;
+                    }
+
+                    var invocation = TryGetInvocationFromRecord(args, taskIdToInvocationMap, projectProperties);
                     if (invocation != null)
                     {
                         invocation.SolutionRoot = Path.GetDirectoryName(binLogFilePath);
                         invocations.Add(invocation);
                     }
                 }
 
-                reader.TargetStarted += TryGetInvocationFromEvent;
+                reader.StatusEventRaised += (object sender, BuildStatusEventArgs e) =>
+                {
+                    if (e?.BuildEventContext?.EvaluationId >= 0 &&
+                        e is ProjectEvaluationFinishedEventArgs projectEvalArgs)
+                    {
+                        // Store the project properties for this project instance
+                        var properties = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
+                        foreach (KeyValuePair<string, string> property in projectEvalArgs.Properties)
+                        {
+                            properties[property.Key] = property.Value;
+                        }
+
+                        projectEvaluationToPropertiesMap[e.BuildEventContext.EvaluationId] = properties;
+                    }
+                };
+
+                reader.ProjectStarted += (object sender, ProjectStartedEventArgs e) =>
+                {
+                    if (e?.BuildEventContext?.EvaluationId >= 0 ||
+                        e?.BuildEventContext?.ProjectInstanceId < 0)
+                    {
+                        projectInstanceToEvaluationMap[e.BuildEventContext.ProjectInstanceId] = e.BuildEventContext.EvaluationId;
+                    }
+                };
+
+                reader.TargetStarted += TryGetInvocationFromEvent;
                 reader.MessageRaised += TryGetInvocationFromEvent;
 
                 reader.Replay(binLogFilePath);
@@ -65,56 +105,61 @@ void TryGetInvocationFromEvent(object sender, BuildEventArgs args)
             return result;
         }
 
-        private static List<CompilerInvocation> ExtractInvocationsFromBuild(string logFilePath)
-        {
-            var build = Microsoft.Build.Logging.StructuredLogger.Serialization.Read(logFilePath);
-            var invocations = new List<CompilerInvocation>();
-            build.VisitAllChildren<Microsoft.Build.Logging.StructuredLogger.Task>(t =>
-            {
-                var invocation = TryGetInvocationFromTask(t);
-                if (invocation != null)
-                {
-                    invocations.Add(invocation);
-                }
-            });
-
-            return invocations;
+        private static List<CompilerInvocation> ExtractInvocationsFromBuild(string logFilePath)
+        {
+            var build = Microsoft.Build.Logging.StructuredLogger.Serialization.Read(logFilePath);
+            var invocations = new List<CompilerInvocation>();
+            build.VisitAllChildren<Microsoft.Build.Logging.StructuredLogger.Task>(t =>
+            {
+                var invocation = TryGetInvocationFromTask(t, build);
+                if (invocation != null)
+                {
+                    invocations.Add(invocation);
+                }
+            });
+
+            return invocations;
         }
 
-        private static CompilerInvocation TryGetInvocationFromRecord(BuildEventArgs args, Dictionary<(int, int), CompilerInvocation> taskIdToInvocationMap)
-        {
-            int targetId = args.BuildEventContext?.TargetId ?? -1;
-            int projectId = args.BuildEventContext?.ProjectInstanceId ?? -1;
-            if (targetId < 0)
-            {
-                return null;
-            }
-
-            var targetStarted = args as TargetStartedEventArgs;
-            if (targetStarted != null && targetStarted.TargetName == "CoreCompile")
-            {
-                var invocation = new CompilerInvocation();
-                taskIdToInvocationMap[(targetId, projectId)] = invocation;
-                invocation.ProjectFilePath = targetStarted.ProjectFile;
-                return null;
-            }
-
-            var commandLine = GetCommandLineFromEventArgs(args, out var language);
-            if (commandLine == null)
-            {
-                return null;
-            }
-
-            CompilerInvocation compilerInvocation;
-            if (taskIdToInvocationMap.TryGetValue((targetId, projectId), out compilerInvocation))
-            {
-                compilerInvocation.Language = language == CompilerKind.CSharp ? LanguageNames.CSharp : LanguageNames.VisualBasic;
-                compilerInvocation.CommandLineArguments = commandLine;
-                Populate(compilerInvocation);
-                taskIdToInvocationMap.Remove((targetId, projectId));
-            }
-
-            return compilerInvocation;
+        private static CompilerInvocation TryGetInvocationFromRecord(BuildEventArgs args, 
+            Dictionary<(int, int), CompilerInvocation> taskIdToInvocationMap,
+            Dictionary<string,string> projectProperties)
+        {
+            int targetId = args.BuildEventContext?.TargetId ?? -1;
+            int projectId = args.BuildEventContext?.ProjectInstanceId ?? -1;
+
+            if (targetId < 0)
+            {
+                return null;
+            }
+
+            if (args is TargetStartedEventArgs targetStarted && targetStarted.TargetName == "CoreCompile")
+            {
+                var invocation = new CompilerInvocation()
+                {
+                    ProjectFilePath = targetStarted.ProjectFile,
+                    ProjectProperties = projectProperties,
+                };
+                taskIdToInvocationMap[(targetId, projectId)] = invocation;
+                return null;
+            }
+
+            var commandLine = GetCommandLineFromEventArgs(args, out var language);
+            if (commandLine == null)
+            {
+                return null;
+            }
+
+            CompilerInvocation compilerInvocation;
+            if (taskIdToInvocationMap.TryGetValue((targetId, projectId), out compilerInvocation))
+            {
+                compilerInvocation.Language = language == CompilerKind.CSharp ? LanguageNames.CSharp : LanguageNames.VisualBasic;
+                compilerInvocation.CommandLineArguments = commandLine;
+                Populate(compilerInvocation);
+                taskIdToInvocationMap.Remove((targetId, projectId));
+            }
+
+            return compilerInvocation;
         }
 
         private static void Populate(CompilerInvocation compilerInvocation)
@@ -125,25 +170,33 @@ private static void Populate(CompilerInvocation compilerInvocation)
             }
         }
 
-        private static CompilerInvocation TryGetInvocationFromTask(Microsoft.Build.Logging.StructuredLogger.Task task)
-        {
-            var name = task.Name;
-            if (name != "Csc" && name != "Vbc" || ((task.Parent as Microsoft.Build.Logging.StructuredLogger.Target)?.Name != "CoreCompile"))
-            {
-                return null;
-            }
-
-            var language = name == "Csc" ? LanguageNames.CSharp : LanguageNames.VisualBasic;
-            var commandLine = task.CommandLineArguments;
-            commandLine = TrimCompilerExeFromCommandLine(commandLine, name == "Csc"
-                ? CompilerKind.CSharp
-                : CompilerKind.VisualBasic);
-            return new CompilerInvocation
-            {
-                Language = language,
-                CommandLineArguments = commandLine,
-                ProjectFilePath = task.GetNearestParent<Microsoft.Build.Logging.StructuredLogger.Project>()?.ProjectFile
-            };
+        private static CompilerInvocation TryGetInvocationFromTask(Microsoft.Build.Logging.StructuredLogger.Task task, Microsoft.Build.Logging.StructuredLogger.Build build)
+        {
+            var name = task.Name;
+            if (name != "Csc" && name != "Vbc" || ((task.Parent as Microsoft.Build.Logging.StructuredLogger.Target)?.Name != "CoreCompile"))
+            {
+                return null;
+            }
+
+            var language = name == "Csc" ? LanguageNames.CSharp : LanguageNames.VisualBasic;
+            var commandLine = task.CommandLineArguments;
+            commandLine = TrimCompilerExeFromCommandLine(commandLine, name == "Csc"
+                ? CompilerKind.CSharp
+                : CompilerKind.VisualBasic);
+
+            // Get the project once and reuse it
+            var project = task.GetNearestParent<Microsoft.Build.Logging.StructuredLogger.Project>();
+
+            var invocation = new CompilerInvocation
+            {
+                Language = language,
+                CommandLineArguments = commandLine,
+                ProjectFilePath = project?.ProjectFile,
+                ProjectProperties = project?.GetEvaluation(build).GetProperties() ?? new Dictionary<string, string>(),
+            };
+
+
+            return invocation;
         }
 
         public static string TrimCompilerExeFromCommandLine(string commandLine, CompilerKind language)

src/SourceBrowser/src/BinLogParser/CompilerInvocation.cs

@@ -9,14 +9,15 @@
 
 namespace Microsoft.SourceBrowser.BinLogParser
 {
-    public class CompilerInvocation
-    {
-        public string ProjectFilePath { get; set; }
-        public string ProjectDirectory => ProjectFilePath == null ? "" : Path.GetDirectoryName(ProjectFilePath);
-        public string OutputAssemblyPath { get; set; }
-        public string CommandLineArguments { get; set; }
-        public string SolutionRoot { get; set; }
-        public IEnumerable<string> TypeScriptFiles { get; set; }
+    public class CompilerInvocation
+    {
+        public string ProjectFilePath { get; set; }
+        public string ProjectDirectory => ProjectFilePath == null ? "" : Path.GetDirectoryName(ProjectFilePath);
+        public string OutputAssemblyPath { get; set; }
+        public string CommandLineArguments { get; set; }
+        public string SolutionRoot { get; set; }
+        public IEnumerable<string> TypeScriptFiles { get; set; }
+        public Dictionary<string, string> ProjectProperties { get; set; } = new Dictionary<string, string>();
 
         public string AssemblyName => Path.GetFileNameWithoutExtension(OutputAssemblyPath);
 

src/SourceBrowser/src/BinLogToSln/BinLogToSln.csproj

@@ -9,10 +9,11 @@
     <VersionPrefix>1.0.1</VersionPrefix>
   </PropertyGroup>
 
-  <ItemGroup>
-    <PackageReference Include="LibGit2Sharp" />
-    <PackageReference Include="Mono.Options" />
-    <PackageReference Include="System.Reflection.Metadata" />
+  <ItemGroup>
+    <PackageReference Include="LibGit2Sharp" />
+    <PackageReference Include="Mono.Options" />
+    <PackageReference Include="NuGet.Frameworks" />
+    <PackageReference Include="System.Reflection.Metadata" />
   </ItemGroup>
 
   <ItemGroup>