Linter for undefined variables

src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs

@@ -15,6 +15,7 @@
 
 using System;
 using System.Collections.Generic;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
@@ -57,7 +58,7 @@ public interface IExpressionEvaluator {
         /// </summary>
         IMember GetValueFromExpression(Expression expr);
 
-        IMember LookupNameInScopes(string name, out IScope scope);
+        IMember LookupNameInScopes(string name, out IScope scope, LookupOptions options = LookupOptions.Normal);
 
         IPythonType GetTypeFromString(string typeString);
 

src/Analysis/Ast/Impl/Analyzer/Evaluation/LookupOptions.cs → src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs

@@ -15,9 +15,9 @@
 
 using System;
 
-namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
+namespace Microsoft.Python.Analysis.Analyzer {
     [Flags]
-    internal enum LookupOptions {
+    public enum LookupOptions {
         None = 0,
         Local,
         Nonlocal,

src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs

@@ -15,6 +15,7 @@
 
 using System.Diagnostics;
 using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Linting;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

@@ -16,12 +16,12 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
 using Microsoft.Python.Analysis.Dependencies;
 using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Linting;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Core;
@@ -267,6 +267,12 @@ private void Analyze(IDependencyChainNode<PythonAnalyzerEntry> node, int version
                 cancellationToken.ThrowIfCancellationRequested();
                 var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval);
 
+                var optionsProvider = _services.GetService<IAnalysisOptionsProvider>();
+                if (optionsProvider?.Options?.LintingEnabled != false) {
+                    var linter = new LinterAggregator();
+                    linter.Lint(analysis, _services);
+                }
+
                 (module as IAnalyzable)?.NotifyAnalysisComplete(analysis);
                 node.Value.TrySetAnalysis(analysis, version, _syncObj);
 

src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs

@@ -0,0 +1,20 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis {
+    public class AnalysisOptions {
+        public bool LintingEnabled { get; set; }
+    }
+}

src/Analysis/Ast/Impl/Definitions/IAnalysisOptionsProvider.cs

@@ -0,0 +1,20 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis {
+    public interface IAnalysisOptionsProvider {
+        AnalysisOptions Options { get; }
+    }
+}

src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs

@@ -22,5 +22,6 @@ public static class ErrorCodes {
         public const string ParameterAlreadySpecified = "parameter-already-specified";
         public const string ParameterMissing = "parameter-missing";
         public const string UnresolvedImport = "unresolved-import";
+        public const string UndefinedVariable = "undefined-variable";
     }
 }

src/Analysis/Ast/Impl/Extensions/SourceSpanExtensions.cs

@@ -0,0 +1,28 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using Microsoft.Python.Core.Text;
+
+namespace Microsoft.Python.Analysis {
+    public static class SourceSpanExtensions {
+        public static bool IsAfter(this SourceSpan span, SourceSpan other) {
+            if (!span.IsValid || !other.IsValid) {
+                return false;
+            }
+            return span.Start > other.Start;
+        }
+    }
+}

src/Analysis/Ast/Impl/Linting/ILinter.cs

@@ -0,0 +1,22 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using Microsoft.Python.Core;
+
+namespace Microsoft.Python.Analysis.Linting {
+    public interface ILinter {
+        void Lint(IDocumentAnalysis analysis, IServiceContainer services);
+    }
+}

src/Analysis/Ast/Impl/Linting/LinterAggregator.cs

@@ -0,0 +1,34 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using Microsoft.Python.Analysis.Linting.UndefinedVariables;
+using Microsoft.Python.Core;
+
+namespace Microsoft.Python.Analysis.Linting {
+    internal sealed class LinterAggregator {
+        private readonly List<ILinter> _linters = new List<ILinter>();
+
+        public LinterAggregator() {
+            // TODO: develop mechanism for dynamic and external linter discovery.
+            _linters.Add(new UndefinedVariablesLinter());
+        }
+        public void Lint(IDocumentAnalysis analysis, IServiceContainer services) {
+            foreach (var l in _linters) {
+                l.Lint(analysis, services);
+            }
+        }
+    }
+}

src/Analysis/Ast/Impl/Linting/LinterWalker.cs

@@ -0,0 +1,47 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Core;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Linting {
+    internal abstract class LinterWalker: PythonWalker {
+        private readonly Stack<IDisposable> _scopeStack = new Stack<IDisposable>();
+
+        protected IDocumentAnalysis Analysis { get; }
+        protected IExpressionEvaluator Eval => Analysis.ExpressionEvaluator;
+        protected IServiceContainer Services { get; }
+
+        protected LinterWalker(IDocumentAnalysis analysis, IServiceContainer services) {
+            Analysis = analysis;
+            Services = services;
+        }
+
+        public override bool Walk(ClassDefinition cd) {
+            _scopeStack.Push(Eval.OpenScope(Analysis.Document, cd));
+            return true;
+        }
+        public override void PostWalk(ClassDefinition cd) => _scopeStack.Pop().Dispose();
+
+        public override bool Walk(FunctionDefinition fd) {
+            _scopeStack.Push(Eval.OpenScope(Analysis.Document, fd));
+            return true;
+        }
+        public override void PostWalk(FunctionDefinition cd) => _scopeStack.Pop().Dispose();
+    }
+}

src/Analysis/Ast/Impl/Linting/UndefinedVariables/AnalysisExtensions.cs

@@ -0,0 +1,31 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Core;
+using Microsoft.Python.Parsing;
+using Microsoft.Python.Parsing.Ast;
+using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes;
+
+namespace Microsoft.Python.Analysis.Linting.UndefinedVariables {
+    internal static class AnalysisExtensions {
+        public static void ReportUndefinedVariable(this IDocumentAnalysis analysis, NameExpression node) {
+            var eval = analysis.ExpressionEvaluator;
+            eval.ReportDiagnostics(analysis.Document.Uri, new DiagnosticsEntry(
+                Resources.UndefinedVariable.FormatInvariant(node.Name),
+                eval.GetLocation(node).Span, ErrorCodes.UndefinedVariable, Severity.Warning));
+        }
+    }
+}

src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs

@@ -0,0 +1,90 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Linting.UndefinedVariables {
+    internal sealed class ComprehensionWalker : PythonWalker {
+        private readonly IDocumentAnalysis _analysis;
+        private readonly HashSet<string> _names = new HashSet<string>();
+        private readonly HashSet<NameExpression> _additionalNameNodes = new HashSet<NameExpression>();
+
+        public ComprehensionWalker(IDocumentAnalysis analysis) {
+            _analysis = analysis;
+        }
+
+        public override bool Walk(GeneratorExpression node) {
+            ProcessComprehension(node, node.Item, node.Iterators);
+            return false;
+        }
+
+        public override bool Walk(ListComprehension node) {
+            ProcessComprehension(node, node.Item, node.Iterators);
+            return false;
+        }
+
+        public override bool Walk(SetComprehension node) {
+            ProcessComprehension(node, node.Item, node.Iterators);
+            return false;
+        }
+
+        public override bool Walk(DictionaryComprehension node) {
+            CollectNames(node);
+            node.Key?.Walk(new ExpressionWalker(_analysis, _names, _additionalNameNodes));
+            node.Value?.Walk(new ExpressionWalker(_analysis, _names, _additionalNameNodes));
+            foreach (var iter in node.Iterators) {
+                iter?.Walk(new ExpressionWalker(_analysis, null, _additionalNameNodes));
+            }
+
+            return true;
+        }
+
+        private void CollectNames(Comprehension c) {
+            var nc = new NameCollectorWalker(_names, _additionalNameNodes);
+            foreach (var cfor in c.Iterators.OfType<ComprehensionFor>()) {
+                cfor.Left?.Walk(nc);
+            }
+        }
+
+        private void ProcessComprehension(Comprehension c, Node item, IEnumerable<ComprehensionIterator> iterators) {
+            CollectNames(c);
+            item?.Walk(new ExpressionWalker(_analysis, _names, _additionalNameNodes));
+            foreach (var iter in iterators) {
+                iter.Walk(new ExpressionWalker(_analysis, null, _additionalNameNodes));
+            }
+        }
+
+        private sealed class NameCollectorWalker : PythonWalker {
+            private readonly HashSet<string> _names;
+            private readonly HashSet<NameExpression> _additionalNameNodes;
+
+            public NameCollectorWalker(HashSet<string> names, HashSet<NameExpression> additionalNameNodes) {
+                _names = names;
+                _additionalNameNodes = additionalNameNodes;
+            }
+
+            public override bool Walk(NameExpression nex) {
+                if (!string.IsNullOrEmpty(nex.Name)) {
+                    _names.Add(nex.Name);
+                    _additionalNameNodes.Add(nex);
+                }
+
+                return false;
+            }
+        }
+    }
+}