Handle on-the-fly search path configuration changes, read search paths from config directly (microsoft#1492)

* Remove SearchPaths and TypeshedPath from Interpreter, move search path logic into module resolution

* Move path processing into config change, with unfortunate extra reload

* Cleanup usings

* Eliminate nulls, since they aren't useful

* Move initialization logic from initialize to initialized, and do an RPC call to get the user search paths instead of reloading, bring back searchPaths for full backwards compat

* Cleanups

* Null checks

* Prevent errors on null settings or when the client doesn't implement workspace/configuration

* Formatting

* Replace IReadOnlyList with ImmutableArray where possible

* Eliminate IReadOnlyList

* Remove unused default/optional parameter

Author: jakebailey

src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs

@@ -23,6 +23,7 @@
 using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core;
+using Microsoft.Python.Core.Collections;
 using Microsoft.Python.Core.Services;
 using Microsoft.Python.Parsing;
 
@@ -43,20 +44,33 @@ private PythonInterpreter(InterpreterConfiguration configuration) {
             LanguageVersion = Configuration.Version.ToLanguageVersion();
         }
 
-        private async Task InitializeAsync(string root, IServiceManager sm, CancellationToken cancellationToken = default) {
+        private async Task InitializeAsync(
+            string root,
+            IServiceManager sm,
+            string typeshedPath,
+            ImmutableArray<string> userConfiguredPaths,
+            CancellationToken cancellationToken
+        ) {
             cancellationToken.ThrowIfCancellationRequested();
 
             sm.AddService(this);
-            _moduleResolution = new MainModuleResolution(root, sm);
-            _stubResolution = new TypeshedResolution(Configuration.TypeshedPath, sm);
-            
+            _moduleResolution = new MainModuleResolution(root, sm, userConfiguredPaths);
+            _stubResolution = new TypeshedResolution(typeshedPath, sm);
+
             await _stubResolution.ReloadAsync(cancellationToken);
             await _moduleResolution.ReloadAsync(cancellationToken);
         }
 
-        public static async Task<IPythonInterpreter> CreateAsync(InterpreterConfiguration configuration, string root, IServiceManager sm, CancellationToken cancellationToken = default) {
+        public static async Task<IPythonInterpreter> CreateAsync(
+            InterpreterConfiguration configuration,
+            string root,
+            IServiceManager sm,
+            string typeshedPath = null,
+            ImmutableArray<string> userConfiguredPaths = default,
+            CancellationToken cancellationToken = default
+        ) {
             var pi = new PythonInterpreter(configuration);
-            await pi.InitializeAsync(root, sm, cancellationToken);
+            await pi.InitializeAsync(root, sm, typeshedPath, userConfiguredPaths, cancellationToken);
 
             // Specialize typing
             TypingModule.Create(sm);

src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs

@@ -15,7 +15,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Threading;
 using Microsoft.Python.Analysis.Caching;
 using Microsoft.Python.Analysis.Core.Interpreter;
 using Microsoft.Python.Analysis.Types;
@@ -25,7 +24,7 @@ namespace Microsoft.Python.Analysis.Modules {
     /// <summary>
     /// Represents module resolution and search subsystem.
     /// </summary>
-    public interface IModuleManagement: IModuleResolution {
+    public interface IModuleManagement : IModuleResolution {
         /// <summary>
         /// Builtins module name.
         /// </summary>
@@ -75,5 +74,7 @@ public interface IModuleManagement: IModuleResolution {
         /// Set of interpreter paths.
         /// </summary>
         ImmutableArray<string> InterpreterPaths { get; }
+
+        bool SetUserConfiguredPaths(ImmutableArray<string> paths);
     }
 }

src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

@@ -41,14 +41,14 @@ internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManage
         private readonly IUIService _ui;
         private IRunningDocumentTable _rdt;
 
-        private ImmutableArray<string> _userPaths = ImmutableArray<string>.Empty;
+        private ImmutableArray<string> _userConfiguredPaths;
 
-        public MainModuleResolution(string root, IServiceContainer services)
+        public MainModuleResolution(string root, IServiceContainer services, ImmutableArray<string> userConfiguredPaths)
             : base(root, services) {
-
             _ui = services.GetService<IUIService>();
+            _userConfiguredPaths = userConfiguredPaths;
         }
-        
+
         public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion);
 
         public IBuiltinsPythonModule BuiltinsModule { get; private set; }
@@ -173,11 +173,11 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) {
 
             await ReloadSearchPaths(cancellationToken);
 
-            PathResolver = new PathResolver(Interpreter.LanguageVersion, Root, InterpreterPaths, _userPaths);
+            PathResolver = new PathResolver(Interpreter.LanguageVersion, Root, InterpreterPaths, UserPaths);
 
             var addedRoots = new HashSet<string> { Root };
             addedRoots.UnionWith(InterpreterPaths);
-            addedRoots.UnionWith(_userPaths);
+            addedRoots.UnionWith(UserPaths);
             ReloadModulePaths(addedRoots);
 
             if (!builtinsIsCreated) {
@@ -198,10 +198,10 @@ private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer serv
 
         private async Task ReloadSearchPaths(CancellationToken cancellationToken = default) {
             var paths = await GetInterpreterSearchPathsAsync(cancellationToken);
-            var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, FileSystem, paths, Configuration.SearchPaths);
+            var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, FileSystem, paths, _userConfiguredPaths);
 
             InterpreterPaths = interpreterPaths.Select(p => p.Path);
-            _userPaths = userPaths.Select(p => p.Path);
+            UserPaths = userPaths.Select(p => p.Path);
 
             if (Log != null) {
                 Log.Log(TraceEventType.Information, "Interpreter search paths:");
@@ -210,12 +210,21 @@ private async Task ReloadSearchPaths(CancellationToken cancellationToken = defau
                 }
 
                 Log.Log(TraceEventType.Information, "User search paths:");
-                foreach (var s in _userPaths) {
+                foreach (var s in UserPaths) {
                     Log.Log(TraceEventType.Information, $"    {s}");
                 }
             }
         }
 
+        public bool SetUserConfiguredPaths(ImmutableArray<string> paths) {
+            if (paths.SequentiallyEquals(_userConfiguredPaths)) {
+                return false;
+            }
+
+            _userConfiguredPaths = paths;
+            return true;
+        }
+
         public bool TryAddModulePath(in string path, in long fileSize, in bool allowNonRooted, out string fullModuleName)
             => PathResolver.TryAddModulePath(path, fileSize, allowNonRooted, out fullModuleName);
 

src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs

@@ -17,11 +17,9 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Threading;
 using Microsoft.Python.Analysis.Caching;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
 using Microsoft.Python.Analysis.Core.Interpreter;
-using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core;
 using Microsoft.Python.Core.Collections;
@@ -55,6 +53,7 @@ protected ModuleResolutionBase(string root, IServiceContainer services) {
         }
 
         public ImmutableArray<string> InterpreterPaths { get; protected set; } = ImmutableArray<string>.Empty;
+        public ImmutableArray<string> UserPaths { get; protected set; } = ImmutableArray<string>.Empty;
 
         /// <summary>
         /// Path resolver providing file resolution in module imports.
@@ -83,7 +82,7 @@ public IPythonModule GetOrLoadModule(string name) {
         public ModulePath FindModule(string filePath) {
             var bestLibraryPath = string.Empty;
 
-            foreach (var p in Configuration.SearchPaths) {
+            foreach (var p in InterpreterPaths.Concat(UserPaths)) {
                 if (PathEqualityComparer.Instance.StartsWith(filePath, p)) {
                     if (p.Length > bestLibraryPath.Length) {
                         bestLibraryPath = p;

src/Analysis/Ast/Test/AnalysisTestBase.cs

@@ -14,7 +14,6 @@
 // permissions and limitations under the License.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Threading;
@@ -66,8 +65,9 @@ protected async Task<IServiceManager> CreateServicesAsync(string root, Interpret
             configuration.AssertInstalled();
             stubCacheFolderPath = stubCacheFolderPath ?? TestData.GetAstAnalysisCachePath(configuration.Version, true);
             Trace.TraceInformation("Cache Path: " + stubCacheFolderPath);
-            configuration.SearchPaths = searchPaths ?? new[] { GetAnalysisTestDataFilesPath() };
-            configuration.TypeshedPath = TestData.GetDefaultTypeshedPath();
+
+            searchPaths = searchPaths ?? new[] { GetAnalysisTestDataFilesPath() };
+            var typeshedPath = TestData.GetDefaultTypeshedPath();
 
             sm = sm ?? CreateServiceManager();
 
@@ -84,7 +84,7 @@ protected async Task<IServiceManager> CreateServicesAsync(string root, Interpret
             sm.AddService(analyzer);
 
             TestLogger.Log(TraceEventType.Information, "Create PythonInterpreter");
-            var interpreter = await PythonInterpreter.CreateAsync(configuration, root, sm);
+            var interpreter = await PythonInterpreter.CreateAsync(configuration, root, sm, typeshedPath, searchPaths.ToImmutableArray());
             sm.AddService(interpreter);
 
             TestLogger.Log(TraceEventType.Information, "Create RunningDocumentTable");

src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs

@@ -62,15 +62,6 @@ private InterpreterConfiguration(IReadOnlyDictionary<string, object> properties)
             } catch (Exception ex) when (ex is ArgumentException || ex is FormatException) {
                 Version = new Version();
             }
-            if (properties.TryGetValue(nameof(SearchPaths), out object o)) {
-                var sp = new List<string>();
-                if (o is string s) {
-                    sp.AddRange(s.Split(';'));
-                } else if (o is IEnumerable<string> ss) {
-                    sp.AddRange(ss);
-                }
-                SearchPaths = sp.ToArray();
-            }
         }
 
         public void WriteToDictionary(IDictionary<string, object> properties) {
@@ -83,7 +74,6 @@ public void WriteToDictionary(IDictionary<string, object> properties) {
             if (Version != null) {
                 properties[nameof(Version)] = Version.ToString();
             }
-            properties[nameof(SearchPaths)] = SearchPaths.ToArray();
         }
 
         /// <summary>
@@ -163,16 +153,6 @@ public void SwitchToFullDescription() {
         /// </summary>
         public string SitePackagesPath { get; }
 
-        /// <summary>
-        /// The fixed search paths of the interpreter.
-        /// </summary>
-        public IReadOnlyList<string> SearchPaths { get; set; } = Array.Empty<string>();
-
-        /// <summary>
-        /// Typeshed root folder.
-        /// </summary>
-        public string TypeshedPath { get; set; }
-
         public static bool operator ==(InterpreterConfiguration x, InterpreterConfiguration y)
             => x?.Equals(y) ?? ReferenceEquals(y, null);
         public static bool operator !=(InterpreterConfiguration x, InterpreterConfiguration y)