Add an additional completion metric into completion_metrics.dart: 'use old vs new relevance rank comparison'

Change-Id: I21eb1447dd6c833870158717031a01d7e8f253c2
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/153485
Commit-Queue: Jaime Wren <[email protected]>
Reviewed-by: Brian Wilkerson <[email protected]>

Author: jwren

pkg/analysis_server/tool/completion_metrics/completion_metrics.dart

@@ -95,6 +95,11 @@ const String SKIP_OLD_RELEVANCE = 'skip-old-relevance';
 /// A flag that causes additional output to be produced.
 const String VERBOSE = 'verbose';
 
+/// A [Counter] to track the performance of the new relevance to the old
+/// relevance.
+Counter oldVsNewComparison =
+    Counter('use old vs new relevance rank comparison');
+
 /// Create a parser that can be used to parse the command-line arguments.
 ArgParser createArgParser() {
   return ArgParser()
@@ -477,6 +482,12 @@ class CompletionMetricsComputer {
       printMetrics(metricsOldMode);
     }
     printMetrics(metricsNewMode);
+
+    print('');
+    print('====================');
+    oldVsNewComparison.printCounterValues();
+    print('====================');
+
     if (verbose) {
       printWorstResults(metricsNewMode);
       printSlowestResults(metricsNewMode);
@@ -485,7 +496,7 @@ class CompletionMetricsComputer {
     return resultCode;
   }
 
-  bool forEachExpectedCompletion(
+  int forEachExpectedCompletion(
       CompletionRequestImpl request,
       MetricsSuggestionListener listener,
       ExpectedCompletion expectedCompletion,
@@ -496,14 +507,14 @@ class CompletionMetricsComputer {
       bool doPrintMissedCompletions) {
     assert(suggestions != null);
 
-    var successfulCompletion;
+    var rank;
 
     var place = placementInSuggestionList(suggestions, expectedCompletion);
 
     metrics.mrrComputer.addRank(place.rank);
 
     if (place.denominator != 0) {
-      successfulCompletion = true;
+      rank = place.rank;
 
       metrics.completionCounter.count('successful');
 
@@ -518,7 +529,7 @@ class CompletionMetricsComputer {
       metrics.insertionLengthTheoretical
           .addValue(expectedCompletion.completion.length - charsBeforeTop);
     } else {
-      successfulCompletion = false;
+      rank = -1;
 
       metrics.completionCounter.count('unsuccessful');
 
@@ -544,7 +555,7 @@ class CompletionMetricsComputer {
         print('');
       }
     }
-    return successfulCompletion;
+    return rank;
   }
 
   void printMetrics(CompletionMetrics metrics) {
@@ -864,7 +875,7 @@ class CompletionMetricsComputer {
             // and results are collected with varying settings for
             // comparison:
 
-            Future<bool> handleExpectedCompletion(
+            Future<int> handleExpectedCompletion(
                 {MetricsSuggestionListener listener,
                 @required CompletionMetrics metrics,
                 @required bool printMissedCompletions,
@@ -910,33 +921,39 @@ class CompletionMetricsComputer {
 
             // First we compute the completions useNewRelevance set to
             // false:
-            var oldRelevanceSucceeded = false;
+            var oldRank;
             if (!skipOldRelevance) {
-              oldRelevanceSucceeded = await handleExpectedCompletion(
+              oldRank = await handleExpectedCompletion(
                   metrics: metricsOldMode,
                   printMissedCompletions: false,
                   useNewRelevance: false);
             }
 
             // And again here with useNewRelevance set to true:
             var listener = MetricsSuggestionListener();
-            var newRelevanceSucceeded = await handleExpectedCompletion(
+            var newRank = await handleExpectedCompletion(
                 listener: listener,
                 metrics: metricsNewMode,
                 printMissedCompletions: verbose,
                 useNewRelevance: true);
 
-            if (!skipOldRelevance &&
-                verbose &&
-                oldRelevanceSucceeded != newRelevanceSucceeded) {
-              if (newRelevanceSucceeded) {
+            if (!skipOldRelevance && newRank != -1 && oldRank != -1) {
+              if (newRank <= oldRank) {
+                oldVsNewComparison.count('new relevance');
+              } else {
+                oldVsNewComparison.count('old relevance');
+              }
+            }
+
+            if (!skipOldRelevance && verbose) {
+              if (newRank > 0 && oldRank < 0) {
                 print('    ===========');
                 print(
                     '    The `useNewRelevance = true` generated a completion that `useNewRelevance = false` did not:');
                 print('    $expectedCompletion');
                 print('    ===========');
                 print('');
-              } else {
+              } else if (newRank < 0 && oldRank > 0) {
                 print('    ===========');
                 print(
                     '    The `useNewRelevance = false` generated a completion that `useNewRelevance = true` did not:');